From 06f141932cd743eaefb0258fecede3ce1d7efe79 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 15 Sep 2021 09:21:19 +0100 Subject: [PATCH 001/161] :bug: Refresh allocation on change in owner (#342) --- code/go/0chain.net/blobbercore/allocation/workers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go index e82dcadb6..e883f262a 100644 --- a/code/go/0chain.net/blobbercore/allocation/workers.go +++ b/code/go/0chain.net/blobbercore/allocation/workers.go @@ -150,7 +150,7 @@ func updateAllocation(ctx context.Context, a *Allocation) { } // if new Tx, then we have to update the allocation - if sa.Tx != a.Tx || sa.Finalized != a.Finalized { + if sa.Tx != a.Tx || sa.OwnerID != a.OwnerID || sa.Finalized != a.Finalized { if a, err = updateAllocationInDB(ctx, a, sa); err != nil { Logger.Error("updating allocation in DB", zap.Error(err)) return @@ -207,6 +207,8 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, // transaction a.Tx = sa.Tx + a.OwnerID = sa.OwnerID + a.OwnerPublicKey = sa.OwnerPublicKey // update fields a.Expiration = sa.Expiration From 24ad186b1b8fd8b4869bc3da8c3b7670084ae595 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 15 Sep 2021 09:21:49 +0100 Subject: [PATCH 002/161] object_operation_handler unit test (#335) --- .../blobbercore/datastore/mock_store.go | 25 +- .../handler/object_operation_handler_test.go | 596 ++++++++++++++++++ code/go/0chain.net/core/transaction/http.go | 3 +- go.mod | 4 +- go.sum | 6 +- 5 files changed, 629 insertions(+), 5 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go diff --git a/code/go/0chain.net/blobbercore/datastore/mock_store.go b/code/go/0chain.net/blobbercore/datastore/mock_store.go index 1fa3c4ef3..c10457bba 100644 --- a/code/go/0chain.net/blobbercore/datastore/mock_store.go +++ b/code/go/0chain.net/blobbercore/datastore/mock_store.go @@ -2,13 +2,36 @@ package datastore import ( "database/sql" + "testing" + "github.com/DATA-DOG/go-sqlmock" + mocket "github.com/selvatico/go-mocket" "github.com/stretchr/testify/require" "gorm.io/driver/postgres" "gorm.io/gorm" - "testing" ) + +func MocketTheStore(t *testing.T, logging bool) { + var err error + + mocket.Catcher.Reset() + mocket.Catcher.Register() + mocket.Catcher.Logging = logging + + dialect := postgres.New(postgres.Config{ + DSN: "mockdb", + DriverName: mocket.DriverName, + PreferSimpleProtocol: true, + }) + + gdb, err := gorm.Open(dialect, new(gorm.Config)) + require.NoError(t, err) + setDB(gdb) +} + +// sqlmock has problems with inserts, so use mocket for tests with inserts +// https://github.com/DATA-DOG/go-sqlmock/issues/118 func MockTheStore(t *testing.T) sqlmock.Sqlmock { var db *sql.DB var mock sqlmock.Sqlmock diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go new file mode 100644 index 000000000..7a861f89c --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -0,0 +1,596 @@ +package handler + +import ( + "bytes" + "context" + "database/sql/driver" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + + "github.com/0chain/gosdk/zboxcore/client" + mocket "github.com/selvatico/go-mocket" + + "github.com/0chain/blobber/code/go/0chain.net/core/node" + + "github.com/0chain/gosdk/zboxcore/fileref" + + "github.com/0chain/gosdk/zboxcore/blockchain" + + "github.com/0chain/blobber/code/go/0chain.net/core/common" + c_common "github.com/0chain/gosdk/core/common" + "github.com/0chain/gosdk/zboxcore/marker" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/stretchr/testify/require" + + "testing" +) + +func TestDownloadFile(t *testing.T) { + const ( + mocketLogging = false + mockBlobberId = "mock_blobber_id" + mockAllocationId = "mock_allocation_id" + mockAllocationTx = "mock_allocation_Tx" + mockRemoteFilePath = "mock/remote/file/path" + mockBlockNumber = 1 + mockEncryptKey = "mock encrypt key" + mockClientWallet = "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}" + mockOwnerWallet = "{\"client_id\":\"5d0229e0141071c1f88785b1faba4b612582f9d446b02e8d893f1e0d0ce92cdc\",\"client_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"keys\":[{\"public_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"private_key\":\"4f8af6fb1098a3817d705aef96db933f31755674b00a5d38bb2439c0a27b0117\"}],\"mnemonics\":\"erode transfer noble civil ridge cloth sentence gauge board wheel sight caution okay sand ranch ice frozen frown grape lion feed fox game zone\",\"version\":\"1.0\",\"date_created\":\"2021-09-04T14:11:06+01:00\"}" + mockReadPrice = int64(0.1 * 1e10) + mockWritePrice = int64(0.5 * 1e10) + mockBigBalance = int64(10000 * 1e10) + mockPoolId = "mock pool id" + ) + ts := time.Now().Add(time.Hour) + var mockLongTimeInFuture = common.Timestamp(ts.Unix()) + common.Timestamp(time.Second*1000) + var mockClient client.Client + require.NoError(t, json.Unmarshal([]byte(mockClientWallet), &mockClient)) + var mockOwner client.Client + require.NoError(t, json.Unmarshal([]byte(mockOwnerWallet), &mockOwner)) + var ( + now = c_common.Timestamp(time.Now().Unix()) + ) + + type ( + blockDownloadRequest struct { + blobber *blockchain.StorageNode + allocationID string + allocationTx string + remotefilepath string + pathHash string + blockNum int64 + encryptedKey string + contentMode string + numBlocks int64 + rxPay bool + } + + parameters struct { + isOwner bool + isCollaborator bool + useAuthTicket bool + attribute common.WhoPays + payerId client.Client + isRevoked bool + isFundedBlobber bool + isFunded0Chain bool + rxPay bool + + // client input from gosdk's BlockDownloadRequest, + inData blockDownloadRequest + + // input from blobber database + allocation allocation.Allocation + } + want struct { + err bool + errMsg string + } + test struct { + name string + parameters parameters + want want + } + ) + node.Self.ID = mockBlobberId + + // reuse code from GOSDK, https://github.com/0chain/gosdk/blob/staging/zboxcore/sdk/blockdownloadworker.go#L150 + var addToForm = func( + t *testing.T, + formWriter *multipart.Writer, + p parameters, + ) *marker.ReadMarker { + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = p.inData.blobber.ID + rm.AllocationID = p.inData.allocationID + rm.OwnerID = mockOwner.ClientID + rm.Timestamp = now + rm.ReadCounter = p.inData.numBlocks + err := rm.Sign() + require.NoError(t, err) + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("path_hash", p.inData.pathHash)) + require.NoError(t, formWriter.WriteField("path", p.inData.remotefilepath)) + if p.inData.rxPay { + require.NoError(t, formWriter.WriteField("rx_pay", "true")) + } + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", p.inData.blockNum))) + require.NoError(t, formWriter.WriteField("num_blocks", fmt.Sprintf("%d", p.inData.numBlocks))) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if p.useAuthTicket { + authTicket := &marker.AuthTicket{ + AllocationID: p.inData.allocationID, + ClientID: client.GetClientID(), + Expiration: int64(time.Duration(now) + 10000*time.Second), + OwnerID: mockOwner.ClientID, + Timestamp: int64(common.Now()), + FilePathHash: p.inData.pathHash, + } + require.NoError(t, client.PopulateClient(mockOwnerWallet, "bls0chain")) + require.NoError(t, authTicket.Sign()) + require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain")) + authTicketBytes, _ := json.Marshal(authTicket) + require.NoError(t, formWriter.WriteField("auth_token", string(authTicketBytes))) + } + if len(p.inData.contentMode) > 0 { + require.NoError(t, formWriter.WriteField("content", p.inData.contentMode)) + } + require.NoError(t, formWriter.Close()) + return rm + } + + makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { + return func(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { + require.New(t) + require.EqualValues(t, scAddress, transaction.STORAGE_CONTRACT_ADDRESS) + switch relativePath { + case "/getReadPoolAllocBlobberStat": + require.False(t, p.isFundedBlobber) + for key, value := range params { + switch key { + case "client_id": + require.EqualValues(t, p.payerId.ClientID, value) + case "allocation_id": + require.EqualValues(t, mockAllocationId, value) + case "blobber_id": + require.EqualValues(t, mockBlobberId, value) + default: + require.Fail(t, "unexpected parameter "+key+" call "+relativePath) + } + } + var pss []*allocation.PoolStat + var funds int64 + if p.isFunded0Chain { + funds = mockBigBalance + } + pss = append(pss, &allocation.PoolStat{ + PoolID: mockPoolId, + Balance: funds, + ExpireAt: mockLongTimeInFuture, + }) + mbytes, err := json.Marshal(&pss) + require.NoError(t, err) + return mbytes, nil + default: + require.Fail(t, "unexpected REST API endpoint call: "+relativePath) + } + return []byte{}, nil + } + } + + setupInMock := func( + t *testing.T, + p parameters, + rm marker.ReadMarker, + ) { + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "allocations" WHERE`, + ).WithArgs( + "mock_allocation_Tx", + ).OneTime().WithReply( + []map[string]interface{}{{ + "id": p.allocation.ID, + "expiration_date": mockLongTimeInFuture, + "owner_id": mockOwner.ClientID, + "owner_public_key": mockOwner.ClientKey, + }}, + ) + + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "terms" WHERE`, + ).WithArgs( + "mock_allocation_id", + ).OneTime().WithReply( + []map[string]interface{}{{ + "blobber_id": mockBlobberId, + "read_price": mockReadPrice, + "write_price": mockWritePrice, + }}, + ) + + attribute, err := json.Marshal(&reference.Attributes{ + WhoPaysForReads: p.attribute, + }) + require.NoError(t, err) + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "reference_objects" WHERE`, + ).WithArgs( + "mock_allocation_id", p.inData.pathHash, + ).WithReply( + []map[string]interface{}{{ + "allocation_id": p.allocation.ID, + "lookup_hash": p.inData.pathHash, + "type": reference.FILE, + "attributes": attribute, + }}, + ) + + var collaboratorRtv int + if p.isCollaborator { + collaboratorRtv = 1 + } + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT count(1) FROM "collaborators" WHERE`, + ).WithArgs( + mockClient.ClientID, + ).WithReply( + []map[string]interface{}{{"count": collaboratorRtv}}, + ) + + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "read_markers" WHERE`, + ).WithCallback(func(par1 string, args []driver.NamedValue) { + require.EqualValues(t, p.payerId.ClientID, args[0].Value) + }).WithArgs( + p.payerId.ClientID, + ).WithReply( + []map[string]interface{}{{ + "client_id": p.allocation.ID, + "redeem_required": false, + }}, + ) + + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "read_markers" WHERE`, + ).WithCallback(func(par1 string, args []driver.NamedValue) { + //require.EqualValues(t, p.payerId.ClientID, args[0].Value) + require.EqualValues(t, client.GetClientID(), args[0].Value) + }).WithReply( + []map[string]interface{}{{ + "client_id": p.allocation.ID, + "redeem_required": false, + }}, + ) + + var funds int64 + if p.isFundedBlobber || p.isFunded0Chain { + funds = mockBigBalance + } + + fundedPool := []map[string]interface{}{{ + "pool_id": "", + "client_id": p.payerId.ClientID, + "blobber_id": mockBlobberId, + "allocation_id": mockAllocationId, + "balance": funds, + "expire_at": mockLongTimeInFuture, + }} + if p.isFundedBlobber { + mocket.Catcher.NewMock().WithCallback(func(par1 string, args []driver.NamedValue) { + require.EqualValues(t, p.payerId.ClientID, args[0].Value) + require.EqualValues(t, mockAllocationId, args[1].Value) + require.EqualValues(t, mockBlobberId, args[2].Value) + }).OneTime().WithQuery(`SELECT * FROM "read_pools" WHERE`).WithReply( + fundedPool, + ) + } else { + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "read_pools" WHERE`, + ).WithCallback(func(_ string, args []driver.NamedValue) { + require.EqualValues(t, p.payerId.ClientID, args[0].Value) + require.EqualValues(t, mockAllocationId, args[1].Value) + require.EqualValues(t, mockBlobberId, args[2].Value) + }).WithReply( + []map[string]interface{}{}, + ) + + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "read_pools" WHERE`, + ).WithCallback(func(_ string, args []driver.NamedValue) { + require.EqualValues(t, p.payerId.ClientID, args[0].Value) + require.EqualValues(t, mockAllocationId, args[1].Value) + require.EqualValues(t, mockBlobberId, args[2].Value) + }).WithReply( + fundedPool, + ) + } + + if p.useAuthTicket { + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "reference_objects" WHERE`, + ).WithCallback(func(_ string, args []driver.NamedValue) { + require.EqualValues(t, p.payerId.ClientID, args[0].Value) + require.EqualValues(t, mockAllocationId, args[1].Value) + require.EqualValues(t, mockBlobberId, args[2].Value) + }).WithArgs(mockAllocationId).WithReply( + []map[string]interface{}{{ + "path": "", + "client_id": p.payerId.ClientID, + "blobber_id": mockBlobberId, + "allocation_id": mockAllocationId, + "balance": mockBigBalance, + "expire_at": mockLongTimeInFuture, + }}, + ) + + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "marketplace_share_info" WHERE`, + ).WithArgs( + mockClient.ClientID, p.inData.pathHash, + ).WithReply( + []map[string]interface{}{{ + "revoked": p.isRevoked, + }}, + ) + } + } + + setupOutMock := func( + t *testing.T, + p parameters, + rm marker.ReadMarker, + ) { + mocket.Catcher.NewMock().WithQuery( + `DELETE FROM "read_pools" WHERE`, + ).WithCallback(func(par1 string, args []driver.NamedValue) { + require.EqualValues(t, p.payerId.ClientID, args[0].Value) + require.EqualValues(t, mockAllocationId, args[1].Value) + require.EqualValues(t, mockBlobberId, args[2].Value) + }).WithID(17) + + var funds int64 + if p.isFunded0Chain || p.isFundedBlobber { + funds = mockBigBalance + } + mocket.Catcher.NewMock().WithQuery( + `INSERT INTO "read_pools"`, + ).WithCallback(func(par1 string, args []driver.NamedValue) { + require.EqualValues(t, mockPoolId, args[0].Value) + require.EqualValues(t, p.payerId.ClientID, args[1].Value) + require.EqualValues(t, mockBlobberId, args[2].Value) + require.EqualValues(t, mockAllocationId, args[3].Value) + require.EqualValues(t, funds, args[4].Value) + require.EqualValues(t, mockLongTimeInFuture, args[5].Value) + }).WithID(23) + + mocket.Catcher.NewMock().WithCallback(func(par1 string, args []driver.NamedValue) { + require.EqualValues(t, client.GetClientID(), args[0].Value) + require.EqualValues(t, client.GetClientPublicKey(), args[1].Value) + require.EqualValues(t, mockBlobberId, args[2].Value) + require.EqualValues(t, mockAllocationId, args[3].Value) + require.EqualValues(t, mockOwner.ClientID, args[4].Value) + require.EqualValues(t, now, args[5].Value) + require.EqualValues(t, p.inData.numBlocks, args[6].Value) + }).WithQuery(`INSERT INTO "read_markers"`).WithID(11) + + mocket.Catcher.NewMock().WithCallback(func(par1 string, args []driver.NamedValue) { + //require.EqualValues(t, p.payerId.ClientKey, args[0].Value) + require.EqualValues(t, client.GetClientPublicKey(), args[0].Value) + require.EqualValues(t, mockBlobberId, args[1].Value) + require.EqualValues(t, mockAllocationId, args[2].Value) + require.EqualValues(t, mockOwner.ClientID, args[3].Value) + require.EqualValues(t, now, args[4].Value) + require.EqualValues(t, p.inData.numBlocks, args[5].Value) + require.EqualValues(t, p.payerId.ClientID, args[7].Value) + }).WithQuery(`UPDATE "read_markers" SET`).WithID(1) + + mocket.Catcher.NewMock().WithQuery(`UPDATE "file_stats" SET`).WithID(1) + } + + setupCtx := func(p parameters) context.Context { + ctx := context.TODO() + ctx = context.WithValue(ctx, constants.CLIENT_CONTEXT_KEY, client.GetClientID()) + ctx = context.WithValue(ctx, constants.ALLOCATION_CONTEXT_KEY, p.inData.allocationTx) + ctx = context.WithValue(ctx, constants.CLIENT_KEY_CONTEXT_KEY, client.GetClientPublicKey()) + + db := datastore.GetStore().GetDB().Begin() + ctx = context.WithValue(ctx, datastore.CONNECTION_CONTEXT_KEY, db) + return ctx + } + + setupRequest := func(p parameters) (*http.Request, *marker.ReadMarker) { + body := new(bytes.Buffer) + formWriter := multipart.NewWriter(body) + rm := addToForm(t, formWriter, p) + req := httptest.NewRequest(http.MethodPost, "/v1/file/download/", body) + require.NoError(t, req.ParseForm()) + req.Header.Add("Content-Type", formWriter.FormDataContentType()) + return req, rm + } + + setupParams := func(p *parameters) { + p.inData = blockDownloadRequest{ + allocationID: mockAllocationId, + allocationTx: mockAllocationTx, + blobber: &blockchain.StorageNode{ID: mockBlobberId}, + remotefilepath: mockRemoteFilePath, + blockNum: mockBlockNumber, + encryptedKey: mockEncryptKey, + contentMode: "", + numBlocks: 10240, + rxPay: p.rxPay, + } + p.allocation = allocation.Allocation{ + ID: mockAllocationId, + Tx: mockAllocationTx, + } + require.True(t, (p.isOwner && !p.isCollaborator && !p.useAuthTicket) || !p.isOwner) + require.True(t, p.attribute == common.WhoPays3rdParty || p.attribute == common.WhoPaysOwner) + p.inData.pathHash = fileref.GetReferenceLookup(p.inData.allocationID, p.inData.remotefilepath) + if p.isOwner || + p.isCollaborator || + (p.attribute == common.WhoPaysOwner && !p.rxPay) { + p.payerId = mockOwner + } else { + p.payerId = mockClient + } + } + + tests := []test{ + { + name: "ok_owner_funded_blobber", + parameters: parameters{ + isOwner: true, + isCollaborator: false, + useAuthTicket: false, + attribute: common.WhoPays3rdParty, + isRevoked: false, + isFundedBlobber: true, + isFunded0Chain: false, + rxPay: false, + }, + }, + { + name: "ok_owner_funded_0chain", + parameters: parameters{ + isOwner: true, + isCollaborator: false, + useAuthTicket: false, + attribute: common.WhoPays3rdParty, + isRevoked: false, + isFundedBlobber: false, + isFunded0Chain: true, + rxPay: false, + }, + }, + { + name: "err_owner_not_funded", + parameters: parameters{ + isOwner: true, + isCollaborator: false, + useAuthTicket: false, + attribute: common.WhoPays3rdParty, + isRevoked: false, + isFundedBlobber: false, + isFunded0Chain: false, + rxPay: false, + }, + want: want{ + err: true, + errMsg: "download_file: pre-redeeming read marker: read_pre_redeem: not enough tokens in client's read pools associated with the allocation->blobber", + }, + }, + { + name: "ok_collaborator", + parameters: parameters{ + isOwner: false, + isCollaborator: true, + useAuthTicket: false, + attribute: common.WhoPays3rdParty, + isRevoked: false, + isFundedBlobber: true, + isFunded0Chain: true, + rxPay: false, + }, + }, + { + name: "ok_authTicket_wp_owner", + parameters: parameters{ + isOwner: false, + isCollaborator: false, + useAuthTicket: true, + attribute: common.WhoPaysOwner, + isRevoked: false, + isFundedBlobber: false, + isFunded0Chain: true, + rxPay: false, + }, + }, + { + name: "ok_authTicket_wp_3rdParty_funded_0chain", + parameters: parameters{ + isOwner: false, + isCollaborator: false, + useAuthTicket: true, + attribute: common.WhoPays3rdParty, + isRevoked: false, + isFundedBlobber: false, + isFunded0Chain: true, + rxPay: false, + }, + }, + { + name: "err_authTicket_wp_3rdParty_revoked", + parameters: parameters{ + isOwner: false, + isCollaborator: false, + useAuthTicket: true, + attribute: common.WhoPays3rdParty, + isRevoked: true, + isFundedBlobber: false, + isFunded0Chain: true, + rxPay: false, + }, + want: want{ + err: true, + errMsg: "client does not have permission to download the file. share revoked", + }, + }, + { + name: "ok_authTicket_wp_owner", + parameters: parameters{ + isOwner: false, + isCollaborator: false, + useAuthTicket: true, + attribute: common.WhoPaysOwner, + isRevoked: false, + isFundedBlobber: false, + isFunded0Chain: true, + rxPay: true, + }, + }, + } + for _, test := range tests { + t.Run(test.name, + func(t *testing.T) { + setupParams(&test.parameters) + if test.parameters.isOwner { + require.NoError(t, client.PopulateClient(mockOwnerWallet, "bls0chain")) + } else { + require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain")) + } + transaction.MakeSCRestAPICall = makeMockMakeSCRestAPICall(t, test.parameters) + request, rm := setupRequest(test.parameters) + datastore.MocketTheStore(t, mocketLogging) + setupInMock(t, test.parameters, *rm) + setupOutMock(t, test.parameters, *rm) + + var sh StorageHandler + _, err := sh.DownloadFile(setupCtx(test.parameters), request) + + require.EqualValues(t, test.want.err, err != nil) + if err != nil { + require.EqualValues(t, test.want.errMsg, err.Error()) + return + } + + }, + ) + } +} diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index c9051bfc0..c54cffc28 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -35,6 +35,7 @@ const ( ) var ErrNoTxnDetail = common.NewError("missing_transaction_detail", "No transaction detail was found on any of the sharders") +var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) = makeSCRestAPICall type SCRestAPIHandler func(response map[string][]byte, numSharders int, err error) @@ -53,7 +54,7 @@ func VerifyTransaction(txnHash string, chain *chain.Chain) (*Transaction, error) } // MakeSCRestAPICall execute api reqeust from sharders, and parse and return result -func MakeSCRestAPICall(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { +func makeSCRestAPICall(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { var resMaxCounterBody []byte var hashMaxCounter int diff --git a/go.mod b/go.mod index ec089afe5..1744366f0 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/0chain/blobber require ( github.com/0chain/errors v1.0.2 - github.com/0chain/gosdk v1.2.82 + github.com/0chain/gosdk v1.2.85-0.20210905084101-4d9d75eb73f2 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/didip/tollbooth v4.0.2+incompatible @@ -20,6 +20,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce github.com/rs/cors v1.8.0 // indirect + github.com/selvatico/go-mocket v1.0.7 github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 @@ -33,6 +34,7 @@ require ( gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d + gorm.io/driver/mysql v0.3.1 gorm.io/driver/postgres v1.0.0 gorm.io/gorm v1.20.4 nhooyr.io/websocket v1.8.7 // indirect diff --git a/go.sum b/go.sum index 5ba678e3d..26206f411 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.2 h1:IIUMeh/qFlqDcyHesjU92CpRMVz9dIQWAtZooqrYinA= github.com/0chain/errors v1.0.2/go.mod h1:5t76jLb56TKfg/K2VD+eUMmNZJ42QsIRI8KzWuztwU4= -github.com/0chain/gosdk v1.2.82 h1:qm0BkPAaiZqKbrgCPf9csaZqTaBkPTrqy9Ehu3Q2D4M= -github.com/0chain/gosdk v1.2.82/go.mod h1:Bl/wsHdlktgXybdzkHPfaf6ATe7mPmew/xF3ki2gskQ= +github.com/0chain/gosdk v1.2.85-0.20210905084101-4d9d75eb73f2 h1:R89Ttt4+Qro5prWnA4YyuHIW3QJRdgIYD90BEG5Iaq4= +github.com/0chain/gosdk v1.2.85-0.20210905084101-4d9d75eb73f2/go.mod h1:Bl/wsHdlktgXybdzkHPfaf6ATe7mPmew/xF3ki2gskQ= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -790,6 +790,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/selvatico/go-mocket v1.0.7 h1:jbVa7RkoOCzBanQYiYF+VWgySHZogg25fOIKkM38q5k= +github.com/selvatico/go-mocket v1.0.7/go.mod h1:7bSWzuNieCdUlanCVu3w0ppS0LvDtPAZmKBIlhoTcp8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= From 512e3ff46cc427d39c5af986457634e5719a09f4 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 23 Sep 2021 09:19:32 +0800 Subject: [PATCH 003/161] ChunkedUpload feature (#344) * feat(upload):refactor code for file upload/update/delete, and add resume upload support * fix(gomod):removed go mod replace * fix(merkle): renamed StreamingMerklHasher with StreamMerklHasher * fix(upload): updated fields on ResumeFileChange * fix(devserver):#145 added devserver feature on blobber and validtor; added scripts and docker-compose.yml for dev * fix(devserver):wrapped transaction.MakeSCRestAPICall with middleware * fix(upload): fixed reload state of allocationChange from db * fix(upload):added to improve build performance;enabled FullSaveAssociations on connectionObj.Save;updated uploadFormData; * fix(devserver):#145 added ./dev.local/cli.sh to run blobber & validator instead of docker * fix(devserver):#145 fixed typo * fix(devserver):#145 added guideline link on README.md * fix(devserver):#145 added build tags was missing * fix(devserver):#145 fixed bug on log_dir, files_dir and db_dir * fix(devserver):#145 added install debuggers * fix(devserver):#145 fixed typo * fix(devserver):#145 fixed bugs on validator * fix(upload):0chain/blobber#111 disable go replace for gosdk * fix(upload):#111 drop devserver part * fix(upload):#111 drop devserver on cli.sh * fix(upload): removed debugging code * fix(devops):0chain/gosdk#107 added github actions for CI/CD * fix(devops):0chain/gosdk#107 fixed build.validator.sh and gosdk repo url * fix(devops):0chain/gosdk#107 fixed gosdk path and disable ci jobs on dayi/* branches * fix(go):removed fmt in debugging * fix(devops):0chain/gosdk#107 remove gosdk before git clone * fix(devops):0chain/gosdk#107 debug git clone issue * fix(devops):0chain/gosdk#107 fixed git clone with branch bug * fix(devops):0chain/gosdk#107 used short GITHUB_SHA instead of latest on docker registry * fix(devops):0chain/gosdk#107 use - instead of latest on docker registry * fix(upload): 0chain/zboxcli#36 update connection size bugs when a chunk is uploaded more than once * fix(git):fixed build scripts about syntax error * fix(dev): added network issue fix script for MacOS * fix(dev): update ip detection * fix(upload&download): 0chain/zboxcli:#37 added chunksize parameter on upload * fix(upload&download): 0chain/zboxcli#36 removed transaction on migrate sql script * fix(liveupload):0chain/zboxcli#54 removed code for debugging * fix(devops):fixed package bug on dockerfile * fix(devops):fixed package bug on dockerfile * fix(devops):fixed package bug on dockerfile * fix(cli):added se-hostname * fix(cli):added change_zcn * fix(cli):skip change zcn if it is blank * feat(cli):fixed validator config * fix(cli):updated logs for cli * fix(challeng):rename ValidationTickets stuff * fix(challenge):remove unsed comment * fix(test):fixed unit test * fix(docs): added comment * fix(core): rename merkletree * fix(upload): normalized names * fix(upload): added ChunkHash in FormData * fix(download): removed ChunkSize from GetFileBlock * fix(upload): Added chunk_size in GetFileBlockForChallenge * fix(challenge):moved syncChallenge to a new worker * fix(challenge):got challenge worked with custom chunk_size and <1M file * fix(challenge):fixed typo * fix(tests):fixed unit tests for CalculateHash with ChunkSize * fix(tests):enabled ci.yml for dayi/* * fix(tests):git clone gosdk instead of gomod * fix(tests):fixed golangci-lint issue * fix(tests):fixed integration-tests * fix(tests):fixed integration-tests for ChunkSize * fix(tests):fixed integration-tests for ChunkSize * fix(upload): refactored AddFileChange * fix(upload): moved context key from blobber/core/contants to gosdk/contants * fix(update): refactored Update with ChunkedUpload * fix(challenge): added recover in challenge worker * fix(tests): fixed unit tests * fix(worker): added recover in workers * fix(worker): added recover in LoadValidationTickets * fix(update): refactored Update with ChunkedUpload * fix(upload): fixed typo issue * fix(git):tidy gomod * fix(datastore): added sqlmock and mocket for Store * fix(datastore): fixed golangci-lint issues * fix(datastore): fixed integration_tests * fix(datastore): fixed integration_tests * fix(datastore): fixed integration_tests * fix(datastore): set timeout_minutes for integration_tests * fix(upload): added BenchmarkUploadFile * fix(golang):upgraded go with 1.16 in github actions * fix(golangci): fixed golangci issues * fix(golang):upgraded go with 1.17 in dockerfile * fix(golang):upgraded docker images with golang:1.17.1-alpine3.14 * fix(golang):upgraded docker images with golang:1.17.1-alpine3.14 * fix(upload): added BenchmarkUploadFileWithNoDisk * fix(devops): test integration-tests * fix(devops): test integration-tests * fix(devops): run integration-tests with non-root * fix(devops): run integration-tests with non-root * fix(devops): fixed Makefile with tag instead space * fix(upload): updated BenchmarkUploadFile * fix(devops): fixed file permission for integration-tests * fix(devops): fixed file permission and removed action-docker-layer-caching from integration-tests * fix(devops): test integration-tests * fix(devops): test integration-tests * fix(devops): test integration-tests * fix(gomod): import gosdk v1.2.87-beat.1 via gomod * fix(gomod): import gosdk v1.2.87-beat.1 via gomod * fix(gomod): import gosdk v1.2.87-beat.1 via gomod in docker build * fix(docker): remove copy .gosdk * fix(gomod): upgraded gosdk with v1.2.88 Co-authored-by: Lz Co-authored-by: Lz --- .github/workflows/benchmark.yml | 51 ++++ .github/workflows/ci.yml | 67 ++++- .github/workflows/dockerize.yml | 219 +++++++++++++++ .github/workflows/tests.yml | 47 ++++ .gitignore | 28 +- Makefile | 3 +- README.md | 2 + code/go/0chain.net/blobber/.gitignore | 2 + code/go/0chain.net/blobber/main.go | 4 +- .../allocation/allocationchange.go | 59 ++-- .../blobbercore/allocation/entity.go | 39 +-- .../blobbercore/allocation/file_changer.go | 22 ++ .../allocation/file_changer_add.go | 117 ++++++++ .../allocation/file_changer_base.go | 91 +++++++ ...tefilechange.go => file_changer_update.go} | 50 +--- .../blobbercore/allocation/newfilechange.go | 66 +++-- .../blobbercore/allocation/protocol.go | 1 + .../blobbercore/allocation/workers.go | 9 +- .../blobbercore/challenge/challenge.go | 138 ++++++++++ .../blobbercore/challenge/protocol.go | 24 +- .../blobbercore/challenge/worker.go | 144 +++------- .../0chain.net/blobbercore/config/config.go | 8 +- .../blobbercore/constants/context_key.go | 12 - .../0chain.net/blobbercore/convert/convert.go | 2 +- .../blobbercore/datastore/mock_store.go | 13 +- .../blobbercore/datastore/mocket.go | 88 ++++++ .../blobbercore/datastore/postgres.go | 68 +++++ .../blobbercore/datastore/sqlmock.go | 83 ++++++ .../0chain.net/blobbercore/datastore/store.go | 83 ++---- .../blobbercore/filestore/fs_store.go | 249 +++++++---------- .../blobbercore/filestore/mock_store.go | 101 +++++++ .../0chain.net/blobbercore/filestore/store.go | 28 +- .../handler/commit_integration_test.go | 4 +- .../blobbercore/handler/file_command.go | 39 +++ .../blobbercore/handler/file_command_add.go | 191 +++++++++++++ .../handler/file_command_delete.go | 77 ++++++ .../handler/file_command_insert.go | 139 ++++++++++ .../handler/file_command_update.go | 200 ++++++++++++++ .../handler/file_command_update_bak.go | 138 ++++++++++ .../0chain.net/blobbercore/handler/handler.go | 18 +- .../handler/handler_integration_tests.go | 10 +- .../blobbercore/handler/handler_test.go | 7 +- .../handler/helper_integration_test.go | 2 +- .../handler/object_operation_handler.go | 217 +++++---------- .../object_operation_handler_bench_test.go | 198 ++++++++++++++ .../handler/object_operation_handler_test.go | 30 +- .../blobbercore/handler/storage_handler.go | 73 ++--- .../handler/storage_handler_bench_test.go | 1 + .../handler/upload_integration_test.go | 5 +- .../0chain.net/blobbercore/handler/worker.go | 169 +++++++----- .../0chain.net/blobbercore/mock/allocation.go | 12 + code/go/0chain.net/blobbercore/mock/buf.go | 14 + code/go/0chain.net/blobbercore/mock/ctx.go | 22 ++ code/go/0chain.net/blobbercore/mock/init.go | 35 +++ .../blobbercore/readmarker/protocol.go | 6 +- .../blobbercore/readmarker/readmarker.go | 56 ++++ .../blobbercore/readmarker/worker.go | 41 +-- .../blobbercore/reference/objectpath.go | 4 + .../0chain.net/blobbercore/reference/ref.go | 11 +- .../blobbercore/reference/referencepath.go | 5 + .../0chain.net/blobbercore/stats/handler.go | 6 +- .../blobbercore/writemarker/entity.go | 3 +- .../blobbercore/writemarker/protocol.go | 9 +- .../blobbercore/writemarker/worker.go | 27 +- .../blobbercore/writemarker/writemarker.go | 42 +++ code/go/0chain.net/core/common/types.go | 3 - code/go/0chain.net/core/node/context.go | 4 +- code/go/0chain.net/core/util/merkle_tree.go | 148 ---------- .../core/util/merkle_tree_interface.go | 53 ---- .../0chain.net/core/util/merkle_tree_test.go | 120 -------- code/go/0chain.net/dev/miner/init.go | 11 + code/go/0chain.net/dev/server.go | 45 +++ code/go/0chain.net/dev/sharder/init.go | 11 + code/go/0chain.net/validator/main.go | 3 +- .../0chain.net/validatorcore/config/config.go | 9 +- .../storage/challenge_handler.go | 1 + .../validatorcore/storage/context.go | 5 +- .../validatorcore/storage/models.go | 17 +- .../validatorcore/storage/models_test.go | 46 ++-- .../validatorcore/storage/protocol.go | 6 +- config/0chain_blobber.yaml | 10 +- config/0chain_validator.yaml | 4 +- dev.local/README.md | 86 ++++++ dev.local/cli.sh | 249 +++++++++++++++++ dev.local/launch.json | 107 ++++++++ docker.aws/build.blobber/Dockerfile | 4 +- docker.aws/build.validator/Dockerfile | 4 +- docker.local/Dockerfile | 6 +- docker.local/Dockerfile.dev | 4 +- .../IntegrationTestsBlobberDockerfile | 4 +- docker.local/ValidatorDockerfile | 6 +- docker.local/ValidatorDockerfile.dev | 4 +- docker.local/base.Dockerfile | 24 ++ docker.local/bin/build.base.sh | 8 + docker.local/bin/build.blobber.sh | 11 +- docker.local/bin/build.validator.sh | 24 ++ docker.local/blobber.Dockerfile | 35 +++ docker.local/docker-compose.yml | 8 + docker.local/validator.Dockerfile | 30 ++ docs/src/stream_hash.puml | 157 +++++++++++ docs/src/upload_flow.puml | 36 +++ go.mod | 20 +- go.sum | 256 ++++++------------ ...15-add-chunk_size-to-reference_objects.sql | 9 + 104 files changed, 3868 insertions(+), 1449 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 .github/workflows/dockerize.yml create mode 100644 .github/workflows/tests.yml create mode 100644 code/go/0chain.net/blobber/.gitignore create mode 100644 code/go/0chain.net/blobbercore/allocation/file_changer.go create mode 100644 code/go/0chain.net/blobbercore/allocation/file_changer_add.go create mode 100644 code/go/0chain.net/blobbercore/allocation/file_changer_base.go rename code/go/0chain.net/blobbercore/allocation/{updatefilechange.go => file_changer_update.go} (57%) create mode 100644 code/go/0chain.net/blobbercore/challenge/challenge.go delete mode 100644 code/go/0chain.net/blobbercore/constants/context_key.go create mode 100644 code/go/0chain.net/blobbercore/datastore/mocket.go create mode 100644 code/go/0chain.net/blobbercore/datastore/postgres.go create mode 100644 code/go/0chain.net/blobbercore/datastore/sqlmock.go create mode 100644 code/go/0chain.net/blobbercore/filestore/mock_store.go create mode 100644 code/go/0chain.net/blobbercore/handler/file_command.go create mode 100644 code/go/0chain.net/blobbercore/handler/file_command_add.go create mode 100644 code/go/0chain.net/blobbercore/handler/file_command_delete.go create mode 100644 code/go/0chain.net/blobbercore/handler/file_command_insert.go create mode 100644 code/go/0chain.net/blobbercore/handler/file_command_update.go create mode 100644 code/go/0chain.net/blobbercore/handler/file_command_update_bak.go create mode 100644 code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go create mode 100644 code/go/0chain.net/blobbercore/handler/storage_handler_bench_test.go create mode 100644 code/go/0chain.net/blobbercore/mock/allocation.go create mode 100644 code/go/0chain.net/blobbercore/mock/buf.go create mode 100644 code/go/0chain.net/blobbercore/mock/ctx.go create mode 100644 code/go/0chain.net/blobbercore/mock/init.go create mode 100644 code/go/0chain.net/blobbercore/readmarker/readmarker.go create mode 100644 code/go/0chain.net/blobbercore/writemarker/writemarker.go delete mode 100644 code/go/0chain.net/core/util/merkle_tree.go delete mode 100644 code/go/0chain.net/core/util/merkle_tree_interface.go delete mode 100644 code/go/0chain.net/core/util/merkle_tree_test.go create mode 100644 code/go/0chain.net/dev/miner/init.go create mode 100644 code/go/0chain.net/dev/server.go create mode 100644 code/go/0chain.net/dev/sharder/init.go create mode 100644 dev.local/README.md create mode 100755 dev.local/cli.sh create mode 100644 dev.local/launch.json create mode 100644 docker.local/base.Dockerfile create mode 100755 docker.local/bin/build.base.sh create mode 100755 docker.local/bin/build.validator.sh create mode 100644 docker.local/blobber.Dockerfile create mode 100644 docker.local/validator.Dockerfile create mode 100644 docs/src/stream_hash.puml create mode 100644 docs/src/upload_flow.puml create mode 100644 sql/15-add-chunk_size-to-reference_objects.sql diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..daddae876 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,51 @@ +name: Benchmark tests + +on: + push: + # branches: [ master ] + pull_request: + # branches: [ master ] + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + + - name: Setup go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '1.16' # The Go version to download (if necessary) and use. + + + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Clone blobber + uses: actions/checkout@v2 + + # - name: Clone gosdk + # run: | + # [ -d $GOSDK ] && rm -rf $GOSDK + # git clone https://github.com/0chain/gosdk.git $GOSDK + # cd $GOSDK + # git checkout $BRANCH + + - name: Run benchmark + run: go test -tags bn256 -benchmem -run="BenchmarkUploadFile*" -bench="BenchmarkUploadFile*" ./... | tee benchmark.txt + + # Run `github-action-benchmark` action + # - name: Push benchmark report + # uses: cnlangzi/github-action-benchmark@v1 + # with: + # name: 0chain/gosdk Benchmark + # tool: 'go' + # benchmark-data-dir-path: ${{ env.TAG }}/bench + # output-file-path: benchmark.txt + # # Personal access token to deploy GitHub Pages branch + # github-token: ${{ secrets.GOSDK }} + # #github-token: ${{ secrets.PERSONAL_GITHUB_TOKEN }} + # # Push and deploy GitHub pages branch automatically + # auto-push: true \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e708cf80..b03989169 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI on: push: + release: types: - published @@ -14,31 +15,76 @@ env: VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} jobs: - test: + integration_test: runs-on: ubuntu-20.04 + timeout-minutes: 40 steps: - - name: Install Go + + - name: Setup go 1.16 uses: actions/setup-go@v2 with: - go-version: 1.13.x - - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.11 - continue-on-error: true + go-version: '1.16' # The Go version to download (if necessary) and use. + + - name: Clone Blobber + uses: actions/checkout@v2 + + # - uses: satackey/action-docker-layer-caching@v0.0.11 + # continue-on-error: true + + + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + # git clone gosdk with same branch + # - name: Clone gosdk + # run: | + # [ -d $GOSDK ] && rm -rf $GOSDK + # git clone https://github.com/0chain/gosdk.git $GOSDK + # cd $GOSDK + # git checkout $BRANCH + - name: Build test environment and run tests run: | sed -i '/#expose_ci_port/c\ ports:\n - "5432:5432"' ./docker.local/b0docker-compose.yml ./docker.local/bin/blobber.init.setup.sh docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 + ./docker.local/bin/build.base.sh ./docker.local/bin/build.blobber.sh + ./docker.local/bin/build.validator.sh cd docker.local/blobber1 ../bin/blobber.start_bls.sh /dev/null & cd ../.. - make integration-tests + + go16=$(which go) + sudo $go16 test ./... -args integration; golangci: name: lint runs-on: ubuntu-latest steps: + - name: Setup go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '1.16' # The Go version to download (if necessary) and use. - uses: actions/checkout@v2 + + + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + # git clone gosdk with same branch + # - name: Clone gosdk + # run: | + # [ -d $GOSDK ] && rm -rf $GOSDK + # git clone https://github.com/0chain/gosdk.git $GOSDK + # cd $GOSDK + # git checkout $BRANCH + - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: @@ -52,18 +98,19 @@ jobs: uses: actions/checkout@master with: fetch-depth: 1 + - name: setup golang if: success() uses: actions/setup-go@v2 with: - go-version: 1.13.x + go-version: 1.16.x - name: run buf commands if: success() run: make local dockerize_blobber: runs-on: ubuntu-20.04 - needs: test + needs: integration_test if: github.event_name == 'release' && github.event.action == 'published' steps: - uses: actions/checkout@v2 @@ -87,7 +134,7 @@ jobs: dockerize_validator: runs-on: ubuntu-20.04 - needs: test + needs: integration_test if: github.event_name == 'release' && github.event.action == 'published' steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml new file mode 100644 index 000000000..b608d9749 --- /dev/null +++ b/.github/workflows/dockerize.yml @@ -0,0 +1,219 @@ +name: Dockerize + +on: + push: + branches: + - dayi/main + - dayi/qa + - dayi/dev + pull_request: + - dayi/main + - dayi/qa + - dayi/dev + # tags: + # - '*' + + +env: + BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} + VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} + +jobs: + build_base: + runs-on: ubuntu-20.04 + steps: + - name: Setup go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '1.16' # The Go version to download (if necessary) and use. + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + - name: Clone blobber + uses: actions/checkout@v2 + + # - name: Clone gosdk + # run: | + # [ -d $GOSDK ] && rm -rf $GOSDK + # git clone https://github.com/0chain/gosdk.git $GOSDK + # cd $GOSDK + # git checkout $BRANCH + + + - name: Build blobber_base + run: ./docker.local/bin/build.base.sh + + - name: Export blobber_base + run: | + mkdir -p /tmp/0chain/ + docker save "blobber_base" > /tmp/0chain/blobber_base.tar + + - name: Upload blobber_base + uses: actions/upload-artifact@v2 + with: + name: blobber_base + path: /tmp/0chain/blobber_base.tar + + build_blobber: + needs: build_base + runs-on: ubuntu-20.04 + steps: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Clone blobber + uses: actions/checkout@v2 + + # - name: Clone gosdk + # run: | + # [ -d $GOSDK ] && rm -rf $GOSDK + # git clone https://github.com/0chain/gosdk.git $GOSDK + # cd $GOSDK + # git checkout $BRANCH + + - name: Download blobber_base + uses: actions/download-artifact@v2 + with: + name: blobber_base + path: /tmp/0chain + + - name: Load blobber_base + run: | + docker load --input /tmp/0chain/blobber_base.tar + + #docker image ls -a + + - name: Build blobber + run: ./docker.local/bin/build.blobber.sh + + - name: Export blobber + run: | + mkdir -p /tmp/0chain/ + docker save "blobber" > /tmp/0chain/blobber.tar + + - name: Upload blobber + uses: actions/upload-artifact@v2 + with: + name: blobber + path: /tmp/0chain/blobber.tar + + publish_blobber: + needs: build_blobber + runs-on: ubuntu-20.04 + steps: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Download blobber + uses: actions/download-artifact@v2 + with: + name: blobber + path: /tmp/0chain + + - name: Load blobber + run: | + docker load --input /tmp/0chain/blobber.tar + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push blobber + run: | + docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG + docker push ${BLOBBER_REGISTRY}:$TAG + + SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) + docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA + docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA + + + + build_validator: + needs: build_base + runs-on: ubuntu-20.04 + steps: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Clone blobber + uses: actions/checkout@v2 + + # - name: Clone gosdk + # run: | + # [ -d $GOSDK ] && rm -rf $GOSDK + # git clone https://github.com/0chain/gosdk.git $GOSDK + # cd $GOSDK + # git checkout $BRANCH + + - name: Download blobber_base + uses: actions/download-artifact@v2 + with: + name: blobber_base + path: /tmp/0chain + + - name: Load blobber_base + run: | + docker load --input /tmp/0chain/blobber_base.tar + + #docker image ls -a + + - name: Build validator + run: ./docker.local/bin/build.validator.sh + + - name: Export validator + run: | + mkdir -p /tmp/0chain/ + docker save "validator" > /tmp/0chain/validator.tar + + - name: Upload validator + uses: actions/upload-artifact@v2 + with: + name: validator + path: /tmp/0chain/validator.tar + + publish_validator: + needs: build_validator + runs-on: ubuntu-20.04 + steps: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Download validator + uses: actions/download-artifact@v2 + with: + name: validator + path: /tmp/0chain + + - name: Load validator + run: | + docker load --input /tmp/0chain/validator.tar + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push validator + run: | + docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG + docker push ${VALIDATOR_REGISTRY}:$TAG + + SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) + docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA + docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..ce181c9ef --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,47 @@ +name: Tests + +on: + push: + branches: + - dayi/main + - dayi/qa + - dayi/dev + pull_request: + - dayi/main + - dayi/qa + - dayi/dev + # tags: + # - '*' + +jobs: + unit_tests: + runs-on: ubuntu-20.04 + steps: + - name: Setup go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '1.16' # The Go version to download (if necessary) and use. + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Clone blobber + uses: actions/checkout@v2 + + # - name: Clone gosdk + # run: | + # [ -d $GOSDK ] && rm -rf $GOSDK + # git clone https://github.com/0chain/gosdk.git $GOSDK + # cd $GOSDK + # git checkout $BRANCH + + - name: Run tests + run: | + cd $GITHUB_WORKSPACE/code/go/0chain.net/ + go test -race -coverprofile=coverage.txt -covermode=atomic ./... + - name: Codecov + run: | + cd $GITHUB_WORKSPACE/code/go/0chain.net/ + bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6c3267d34..7e5599120 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +coverage.txt .DS_Store .idea/ .vscode/ +__debug_bin **/client !**/client/main.go **/docker.local/blobber* @@ -9,28 +11,6 @@ **/blobber/blobber/files **/blobber/blobber/data **/pkg/ -docker.local/blobber1/data/badgerdb/blobberstate/000000.vlog -docker.local/blobber1/data/badgerdb/blobberstate/LOCK -docker.local/blobber1/files/7a7/a9b/206/177a1b97ac960fa1ed4f5d5bc925fd1821cf69d387165f402275bdf/refs/123/6e9/a3b/6c8bbc889679c68af4372b7d68e2f2343c03ae7aaa4b25fbb8257c3 -docker.local/blobber1/files/bdf/b41/6eb/0cf7bd794d8c27fd840519ceb1d82a2207270b70f3f7c70b288c517/refs/123/6e9/a3b/6c8bbc889679c68af4372b7d68e2f2343c03ae7aaa4b25fbb8257c3 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/03d/67d/23a/b7ecc9af4f5bbfef03892ca2ec83138 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/164/731/b66/0a01c9212d23b19c8d6d9b5ad04d95d -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/262/451/a50/c1940645c1428309a4eb50349843e33 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/29a/f44/4b3/acad044ba91d6c6b2bff2f5156c6d36 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/484/7a7/d69/b0d4e90ed6c3ae8ea9e07afbda4dfa9 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/530/aaa/fb4/749ba793413bfd63bfa681d1ef76843 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/740/6ce/091/6703b9620afdd7a568d42fc2980f02d -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/873/b24/50a/26cf8f5c6e7fc0435351272603df08f -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/967/e7d/1f1/f472f222c60e31ce5a95ca92749d7d7 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/a66/228/e80/db0cbfd58778557fb6592da86837e3e -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/abc/954/6e7/687b12ef616ded07546fea91ba73042 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/bdd/772/f67/d87ee8bf3560d724ea030bf49790156 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/c19/186/b15/8a7f8c38530fec679cbdad1e43b1a3c -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/c2b/e1e/7c1/baec0d6299391c115b39187b249c15a -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/dd5/f91/f9f/2fb2b6c93e400f3cdc1e29654bb397d -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/objects/f4c/044/59a/1c3f87be1184b5b495592408f6708fe -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/refs/123/6e9/a3b/6c8bbc889679c68af4372b7d68e2f2343c03ae7aaa4b25fbb8257c3 -docker.local/blobber1/files/e55/003/7d3/f56519dff06156f16a19a40389414f94cfd4c3a548f46c218cb484e/refs/388/12d/bd3/b084ff297706d7bd307996c3363c2df53acc49c97b83201f8dc7bd9 -docker.local/blobber1/data/badgerdb/blobberstate/MANIFEST +dev.local/data +out/ -dev.local/data \ No newline at end of file diff --git a/Makefile b/Makefile index 5c9dc25ed..aa62d38b0 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,10 @@ test: lint: golangci-lint run --timeout 2m0s; + .PHONY: integration-tests integration-tests: - sudo go test ./... -args integration; + go test ./... -args integration; diff --git a/README.md b/README.md index d5281f6d8..53e642a23 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ - [Run blobber on ec2 / vm / bare metal over https](https://github.com/0chain/blobber/blob/master/https/README.md) +- [Blobber local development guideline](dev.local/README.md) + ## Initial Setup diff --git a/code/go/0chain.net/blobber/.gitignore b/code/go/0chain.net/blobber/.gitignore new file mode 100644 index 000000000..9e5fb362b --- /dev/null +++ b/code/go/0chain.net/blobber/.gitignore @@ -0,0 +1,2 @@ +blobber +/config \ No newline at end of file diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index e89aebf01..f5ae1f061 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -350,11 +350,13 @@ func main() { portString := flag.String("port", "", "port") grpcPortString := flag.String("grpc_port", "", "grpc_port") hostname := flag.String("hostname", "", "hostname") + configDir := flag.String("config_dir", "./config", "config_dir") flag.Parse() config.SetupDefaultConfig() - config.SetupConfig("./config") + + config.SetupConfig(*configDir) config.Configuration.DeploymentMode = byte(*deploymentMode) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 554cf3151..e7bc51ebc 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -8,21 +8,12 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/constants" "go.uber.org/zap" "gorm.io/gorm" ) -const ( - INSERT_OPERATION = "insert" - DELETE_OPERATION = "delete" - UPDATE_OPERATION = "update" - RENAME_OPERATION = "rename" - COPY_OPERATION = "copy" - UPDATE_ATTRS_OPERATION = "update_attrs" - CREATEDIR_OPERATION = "createdir" -) - const ( NewConnection = 0 InProgressConnection = 1 @@ -32,6 +23,7 @@ const ( var OperationNotApplicable = common.NewError("operation_not_valid", "Not an applicable operation") +// AllocationChangeProcessor request transaction of file operation. it is president in postgres, and can be rebuilt for next http reqeust(eg CommitHandler) type AllocationChangeProcessor interface { CommitToFileStore(ctx context.Context) error DeleteTempFile() error @@ -68,22 +60,31 @@ func (AllocationChange) TableName() string { return "allocation_changes" } +func (change *AllocationChange) Save(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + + return db.Save(change).Error +} + +// GetAllocationChanges reload connection's changes in allocation from postgres. +// 1. update connection's status with NewConnection if connection_id is not found in postgres +// 2. mark as NewConnection if connection_id is marked as DeleteConnection func GetAllocationChanges(ctx context.Context, connectionID string, allocationID string, clientID string) (*AllocationChangeCollector, error) { cc := &AllocationChangeCollector{} db := datastore.GetStore().GetTransaction(ctx) - err := db.Where(&AllocationChangeCollector{ - ConnectionID: connectionID, - AllocationID: allocationID, - ClientID: clientID, - }).Not(&AllocationChangeCollector{ - Status: DeletedConnection, - }).Preload("Changes").First(cc).Error + err := db.Where("connection_id = ? and allocation_id = ? and client_id = ? and status <> ?", + connectionID, + allocationID, + clientID, + DeletedConnection, + ).Preload("Changes").First(cc).Error if err == nil { cc.ComputeProperties() return cc, nil } + // It is a bug when connetion_id was marked as DeletedConnection if errors.Is(err, gorm.ErrRecordNotFound) { cc.ConnectionID = connectionID cc.AllocationID = allocationID @@ -105,31 +106,31 @@ func (cc *AllocationChangeCollector) Save(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) if cc.Status == NewConnection { cc.Status = InProgressConnection - err := db.Create(cc).Error - return err - } else { - err := db.Save(cc).Error - return err + return db.Create(cc).Error } + + return db.Save(cc).Error } +// ComputeProperties unmarshal all ChangeProcesses from postgres func (cc *AllocationChangeCollector) ComputeProperties() { cc.AllocationChanges = make([]AllocationChangeProcessor, 0, len(cc.Changes)) for _, change := range cc.Changes { var acp AllocationChangeProcessor switch change.Operation { - case INSERT_OPERATION: + case constants.FileOperationInsert: acp = new(NewFileChange) - case UPDATE_OPERATION: - acp = new(UpdateFileChange) - case DELETE_OPERATION: + case constants.FileOperationUpdate: + acp = new(UpdateFileChanger) + case constants.FileOperationDelete: acp = new(DeleteFileChange) - case RENAME_OPERATION: + case constants.FileOperationRename: acp = new(RenameFileChange) - case COPY_OPERATION: + case constants.FileOperationCopy: acp = new(CopyFileChange) - case UPDATE_ATTRS_OPERATION: + case constants.FileOperationUpdateAttrs: acp = new(AttributesChange) + } if acp == nil { diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 463b65239..6196ead9b 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -19,27 +19,28 @@ const ( ) type Allocation struct { - ID string `gorm:"column:id;primary_key"` - Tx string `gorm:"column:tx"` - TotalSize int64 `gorm:"column:size"` - UsedSize int64 `gorm:"column:used_size"` - OwnerID string `gorm:"column:owner_id"` - OwnerPublicKey string `gorm:"column:owner_public_key"` - RepairerID string `gorm:"column:repairer_id"`// experimental / blobber node id - PayerID string `gorm:"column:payer_id"` // optional / client paying for all r/w ops - Expiration common.Timestamp `gorm:"column:expiration_date"` - AllocationRoot string `gorm:"column:allocation_root"` - BlobberSize int64 `gorm:"column:blobber_size"` - BlobberSizeUsed int64 `gorm:"column:blobber_size_used"` - LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker"` - IsRedeemRequired bool `gorm:"column:is_redeem_required"` - TimeUnit time.Duration `gorm:"column:time_unit"` - IsImmutable bool `gorm:"is_immutable"` + ID string `gorm:"column:id;primary_key"` + Tx string `gorm:"column:tx"` + TotalSize int64 `gorm:"column:size"` + UsedSize int64 `gorm:"column:used_size"` + OwnerID string `gorm:"column:owner_id"` + OwnerPublicKey string `gorm:"column:owner_public_key"` + RepairerID string `gorm:"column:repairer_id"` // experimental / blobber node id + PayerID string `gorm:"column:payer_id"` // optional / client paying for all r/w ops + Expiration common.Timestamp `gorm:"column:expiration_date"` + // AllocationRoot allcation_root of last write_marker + AllocationRoot string `gorm:"column:allocation_root"` + BlobberSize int64 `gorm:"column:blobber_size"` + BlobberSizeUsed int64 `gorm:"column:blobber_size_used"` + LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker"` + IsRedeemRequired bool `gorm:"column:is_redeem_required"` + TimeUnit time.Duration `gorm:"column:time_unit"` + IsImmutable bool `gorm:"is_immutable"` // Ending and cleaning - CleanedUp bool `gorm:"column:cleaned_up"` - Finalized bool `gorm:"column:finalized"` + CleanedUp bool `gorm:"column:cleaned_up"` + Finalized bool `gorm:"column:finalized"` // Has many terms - Terms []*Terms `gorm:"-"` + Terms []*Terms `gorm:"-"` } func (Allocation) TableName() string { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer.go b/code/go/0chain.net/blobbercore/allocation/file_changer.go new file mode 100644 index 000000000..705aef90b --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/file_changer.go @@ -0,0 +1,22 @@ +package allocation + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" +) + +// FileChanger file change processor +type FileChanger interface { + // ProcessChange process change and save them on reference_objects + ProcessChange(ctx context.Context, + change *AllocationChange, allocationRoot string) (*reference.Ref, error) + // Marshal marshal change as JSON string + Marshal() (string, error) + // Unmarshal unmarshal change from JSON string + Unmarshal(input string) error + // DeleteTempFile delete temp file and thumbnail from disk + DeleteTempFile() error + // CommitToFileStore move temp file and thumbnail from temp dir to persistent folder + CommitToFileStore(ctx context.Context) error +} diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_add.go b/code/go/0chain.net/blobbercore/allocation/file_changer_add.go new file mode 100644 index 000000000..cd6dcc249 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_add.go @@ -0,0 +1,117 @@ +package allocation + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" + + "github.com/0chain/blobber/code/go/0chain.net/core/common" +) + +// AddFileChanger file change processor for continuous upload in INIT/APPEND/FINALIZE +type AddFileChanger struct { + BaseFileChanger +} + +// ProcessChange update references, and create a new FileRef +func (nf *AddFileChanger) ProcessChange(ctx context.Context, + change *AllocationChange, allocationRoot string) (*reference.Ref, error) { + + path, _ := filepath.Split(nf.Path) + path = filepath.Clean(path) + tSubDirs := reference.GetSubDirsFromPath(path) + + rootRef, err := reference.GetReferencePath(ctx, nf.AllocationID, nf.Path) + if err != nil { + return nil, err + } + + dirRef := rootRef + treelevel := 0 + for { + found := false + for _, child := range dirRef.Children { + if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) { + if child.Name == tSubDirs[treelevel] { + dirRef = child + found = true + break + } + } + } + if found { + treelevel++ + continue + } + if len(tSubDirs) > treelevel { + newRef := reference.NewDirectoryRef() + newRef.AllocationID = dirRef.AllocationID + newRef.Path = "/" + strings.Join(tSubDirs[:treelevel+1], "/") + newRef.ParentPath = "/" + strings.Join(tSubDirs[:treelevel], "/") + newRef.Name = tSubDirs[treelevel] + newRef.LookupHash = reference.GetReferenceLookup(dirRef.AllocationID, newRef.Path) + dirRef.AddChild(newRef) + dirRef = newRef + treelevel++ + continue + } else { + break + } + } + + var newFile = reference.NewFileRef() + newFile.ActualFileHash = nf.ActualHash + newFile.ActualFileSize = nf.ActualSize + newFile.AllocationID = dirRef.AllocationID + newFile.ContentHash = nf.Hash + newFile.CustomMeta = nf.CustomMeta + newFile.MerkleRoot = nf.MerkleRoot + newFile.Name = nf.Filename + newFile.ParentPath = dirRef.Path + newFile.Path = nf.Path + newFile.LookupHash = reference.GetReferenceLookup(dirRef.AllocationID, nf.Path) + newFile.Size = nf.Size + newFile.MimeType = nf.MimeType + newFile.WriteMarker = allocationRoot + newFile.ThumbnailHash = nf.ThumbnailHash + newFile.ThumbnailSize = nf.ThumbnailSize + newFile.ActualThumbnailHash = nf.ActualThumbnailHash + newFile.ActualThumbnailSize = nf.ActualThumbnailSize + newFile.EncryptedKey = nf.EncryptedKey + newFile.ChunkSize = nf.ChunkSize + + if err = newFile.SetAttributes(&nf.Attributes); err != nil { + return nil, common.NewErrorf("process_new_file_change", + "setting file attributes: %v", err) + } + + dirRef.AddChild(newFile) + if _, err := rootRef.CalculateHash(ctx, true); err != nil { + return nil, err + } + stats.NewFileCreated(ctx, newFile.ID) + return rootRef, nil +} + +// Marshal marshal and change to persistent to postgres +func (nf *AddFileChanger) Marshal() (string, error) { + ret, err := json.Marshal(nf) + if err != nil { + return "", err + } + return string(ret), nil +} + +// Unmarshal reload and unmarshal change from allocation_changes.input on postgres +func (nf *AddFileChanger) Unmarshal(input string) error { + if err := json.Unmarshal([]byte(input), nf); err != nil { + return err + } + + return util.UnmarshalValidation(nf) +} diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go new file mode 100644 index 000000000..7fa347854 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -0,0 +1,91 @@ +package allocation + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" +) + +// BaseFileChanger base file change processor +type BaseFileChanger struct { + //client side: unmarshal them from 'updateMeta'/'uploadMeta' + ConnectionID string `json:"connection_id" validation:"required"` + //client side: + Filename string `json:"filename" validation:"required"` + //client side: + Path string `json:"filepath" validation:"required"` + //client side: + ActualHash string `json:"actual_hash,omitempty" validation:"required"` + //client side: + ActualSize int64 `json:"actual_size,omitempty" validation:"required"` + //client side: + ActualThumbnailSize int64 `json:"actual_thumb_size"` + //client side: + ActualThumbnailHash string `json:"actual_thumb_hash"` + //client side: + MimeType string `json:"mimetype,omitempty"` + //client side: + Attributes reference.Attributes `json:"attributes,omitempty"` + //client side: + MerkleRoot string `json:"merkle_root,omitempty"` + + //server side: update them by ChangeProcessor + AllocationID string `json:"allocation_id"` + //client side: + Hash string `json:"content_hash,omitempty"` + Size int64 `json:"size"` + //server side: + ThumbnailHash string `json:"thumbnail_content_hash,omitempty"` + ThumbnailSize int64 `json:"thumbnail_size"` + ThumbnailFilename string `json:"thumbnail_filename"` + + EncryptedKey string `json:"encrypted_key,omitempty"` + CustomMeta string `json:"custom_meta,omitempty"` + + ChunkSize int64 `json:"chunk_size,omitempty"` // the size of achunk. 64*1024 is default + IsFinal bool `json:"is_final,omitempty"` // current chunk is last or not + + ChunkIndex int `json:"chunk_index,omitempty"` // the seq of current chunk. all chunks MUST be uploaded one by one because of CompactMerkleTree + ChunkHash string `json:"chunk_hash,omitempty"` + UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to +} + +func (nf *BaseFileChanger) DeleteTempFile() error { + fileInputData := &filestore.FileInputData{} + fileInputData.Name = nf.Filename + fileInputData.Path = nf.Path + fileInputData.Hash = nf.Hash + err := filestore.GetFileStore().DeleteTempFile(nf.AllocationID, fileInputData, nf.ConnectionID) + if nf.ThumbnailSize > 0 { + fileInputData := &filestore.FileInputData{} + fileInputData.Name = nf.ThumbnailFilename + fileInputData.Path = nf.Path + fileInputData.Hash = nf.ThumbnailHash + err = filestore.GetFileStore().DeleteTempFile(nf.AllocationID, fileInputData, nf.ConnectionID) + } + return err +} + +func (nfch *BaseFileChanger) CommitToFileStore(ctx context.Context) error { + fileInputData := &filestore.FileInputData{} + fileInputData.Name = nfch.Filename + fileInputData.Path = nfch.Path + fileInputData.Hash = nfch.Hash + _, err := filestore.GetFileStore().CommitWrite(nfch.AllocationID, fileInputData, nfch.ConnectionID) + if err != nil { + return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) + } + if nfch.ThumbnailSize > 0 { + fileInputData := &filestore.FileInputData{} + fileInputData.Name = nfch.ThumbnailFilename + fileInputData.Path = nfch.Path + fileInputData.Hash = nfch.ThumbnailHash + _, err := filestore.GetFileStore().CommitWrite(nfch.AllocationID, fileInputData, nfch.ConnectionID) + if err != nil { + return common.NewError("file_store_error", "Error committing thumbnail to file store. "+err.Error()) + } + } + return nil +} diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go similarity index 57% rename from code/go/0chain.net/blobbercore/allocation/updatefilechange.go rename to code/go/0chain.net/blobbercore/allocation/file_changer_update.go index d06006f12..0119e9bb7 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -5,7 +5,6 @@ import ( "encoding/json" "path/filepath" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" @@ -15,11 +14,11 @@ import ( "go.uber.org/zap" ) -type UpdateFileChange struct { - NewFileChange +type UpdateFileChanger struct { + BaseFileChanger } -func (nf *UpdateFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { +func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { path, _ := filepath.Split(nf.Path) path = filepath.Clean(path) @@ -74,6 +73,7 @@ func (nf *UpdateFileChange) ProcessChange(ctx context.Context, change *Allocatio existingRef.ActualThumbnailHash = nf.ActualThumbnailHash existingRef.ActualThumbnailSize = nf.ActualThumbnailSize existingRef.EncryptedKey = nf.EncryptedKey + existingRef.ChunkSize = nf.ChunkSize if err = existingRef.SetAttributes(&nf.Attributes); err != nil { return nil, common.NewErrorf("process_update_file_change", @@ -85,7 +85,7 @@ func (nf *UpdateFileChange) ProcessChange(ctx context.Context, change *Allocatio return rootRef, err } -func (nf *UpdateFileChange) Marshal() (string, error) { +func (nf *UpdateFileChanger) Marshal() (string, error) { ret, err := json.Marshal(nf) if err != nil { return "", err @@ -93,48 +93,10 @@ func (nf *UpdateFileChange) Marshal() (string, error) { return string(ret), nil } -func (nf *UpdateFileChange) Unmarshal(input string) error { +func (nf *UpdateFileChanger) Unmarshal(input string) error { if err := json.Unmarshal([]byte(input), nf); err != nil { return err } return util.UnmarshalValidation(nf) } - -func (nf *UpdateFileChange) DeleteTempFile() error { - fileInputData := &filestore.FileInputData{} - fileInputData.Name = nf.Filename - fileInputData.Path = nf.Path - fileInputData.Hash = nf.Hash - err := filestore.GetFileStore().DeleteTempFile(nf.AllocationID, fileInputData, nf.ConnectionID) - if nf.ThumbnailSize > 0 { - fileInputData := &filestore.FileInputData{} - fileInputData.Name = nf.ThumbnailFilename - fileInputData.Path = nf.Path - fileInputData.Hash = nf.ThumbnailHash - err = filestore.GetFileStore().DeleteTempFile(nf.AllocationID, fileInputData, nf.ConnectionID) - } - return err -} - -func (nfch *UpdateFileChange) CommitToFileStore(ctx context.Context) error { - fileInputData := &filestore.FileInputData{} - fileInputData.Name = nfch.Filename - fileInputData.Path = nfch.Path - fileInputData.Hash = nfch.Hash - _, err := filestore.GetFileStore().CommitWrite(nfch.AllocationID, fileInputData, nfch.ConnectionID) - if err != nil { - return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) - } - if nfch.ThumbnailSize > 0 { - fileInputData := &filestore.FileInputData{} - fileInputData.Name = nfch.ThumbnailFilename - fileInputData.Path = nfch.Path - fileInputData.Hash = nfch.ThumbnailHash - _, err := filestore.GetFileStore().CommitWrite(nfch.AllocationID, fileInputData, nfch.ConnectionID) - if err != nil { - return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) - } - } - return nil -} diff --git a/code/go/0chain.net/blobbercore/allocation/newfilechange.go b/code/go/0chain.net/blobbercore/allocation/newfilechange.go index f3761f158..23d33fbe8 100644 --- a/code/go/0chain.net/blobbercore/allocation/newfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/newfilechange.go @@ -11,36 +11,45 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/gosdk/constants" ) type NewFileChange struct { - ConnectionID string `json:"connection_id" validation:"required"` - AllocationID string `json:"allocation_id"` - Filename string `json:"filename" validation:"required"` - ThumbnailFilename string `json:"thumbnail_filename"` - Path string `json:"filepath" validation:"required"` - Size int64 `json:"size"` - Hash string `json:"content_hash,omitempty"` - ThumbnailSize int64 `json:"thumbnail_size"` - ThumbnailHash string `json:"thumbnail_content_hash,omitempty"` - MerkleRoot string `json:"merkle_root,omitempty"` - ActualHash string `json:"actual_hash,omitempty" validation:"required"` - ActualSize int64 `json:"actual_size,omitempty" validation:"required"` - ActualThumbnailSize int64 `json:"actual_thumb_size"` - ActualThumbnailHash string `json:"actual_thumb_hash"` - MimeType string `json:"mimetype,omitempty"` - EncryptedKey string `json:"encrypted_key,omitempty"` - CustomMeta string `json:"custom_meta,omitempty"` - Attributes reference.Attributes `json:"attributes,omitempty"` - - // IsResumable the request is resumable upload - IsResumable bool `json:"is_resumable,omitempty"` - // UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. - UploadLength int64 `json:"upload_length,omitempty"` - // Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. - UploadOffset int64 `json:"upload_offset,omitempty"` - // IsFinal the request is final chunk - IsFinal bool `json:"is_final,omitempty"` + //client side: unmarshal them from 'updateMeta'/'uploadMeta' + ConnectionID string `json:"connection_id" validation:"required"` + //client side: + Filename string `json:"filename" validation:"required"` + //client side: + Path string `json:"filepath" validation:"required"` + //client side: + ActualHash string `json:"actual_hash,omitempty" validation:"required"` + //client side: + ActualSize int64 `json:"actual_size,omitempty" validation:"required"` + //client side: + ActualThumbnailSize int64 `json:"actual_thumb_size"` + //client side: + ActualThumbnailHash string `json:"actual_thumb_hash"` + //client side: + MimeType string `json:"mimetype,omitempty"` + //client side: + Attributes reference.Attributes `json:"attributes,omitempty"` + //client side: + MerkleRoot string `json:"merkle_root,omitempty"` + + //server side: update them by ChangeProcessor + AllocationID string `json:"allocation_id"` + //client side: + Hash string `json:"content_hash,omitempty"` + Size int64 `json:"size"` + //server side: + ThumbnailHash string `json:"thumbnail_content_hash,omitempty"` + ThumbnailSize int64 `json:"thumbnail_size"` + ThumbnailFilename string `json:"thumbnail_filename"` + + EncryptedKey string `json:"encrypted_key,omitempty"` + CustomMeta string `json:"custom_meta,omitempty"` + + ChunkSize int64 `json:"chunk_size,omitempty"` // the size of achunk. 64*1024 is default } func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, allocationRoot string) (*reference.Ref, error) { @@ -107,7 +116,7 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a func (nf *NewFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { - if change.Operation == CREATEDIR_OPERATION { + if change.Operation == constants.FileOperationCreateDir { err := nf.Unmarshal(change.Input) if err != nil { return nil, err @@ -177,6 +186,7 @@ func (nf *NewFileChange) ProcessChange(ctx context.Context, newFile.ActualThumbnailHash = nf.ActualThumbnailHash newFile.ActualThumbnailSize = nf.ActualThumbnailSize newFile.EncryptedKey = nf.EncryptedKey + newFile.ChunkSize = nf.ChunkSize if err = newFile.SetAttributes(&nf.Attributes); err != nil { return nil, common.NewErrorf("process_new_file_change", diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 5bd71ca5e..25d03d9bb 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -49,6 +49,7 @@ func (a *Allocation) LoadTerms(ctx context.Context) (err error) { return // found in DB } +// VerifyAllocationTransaction try to get allocation from postgres.if it doesn't exists, get it from sharders, and insert it into postgres. func VerifyAllocationTransaction(ctx context.Context, allocationTx string, readonly bool) (a *Allocation, err error) { diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go index e883f262a..b57ef59fc 100644 --- a/code/go/0chain.net/blobbercore/allocation/workers.go +++ b/code/go/0chain.net/blobbercore/allocation/workers.go @@ -11,6 +11,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/zboxutil" "gorm.io/gorm" @@ -73,6 +74,12 @@ func waitOrQuit(ctx context.Context, d time.Duration) (quit bool) { func updateWork(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + Logger.Error("[recover] updateWork", zap.Any("err", r)) + } + }() + var ( allocs []*Allocation count int64 @@ -364,7 +371,7 @@ func deleteFile(ctx context.Context, path string, change.ConnectionID = conn.ConnectionID change.Size = 0 - deleteSize - change.Operation = DELETE_OPERATION + change.Operation = constants.FileOperationDelete var dfc = &DeleteFileChange{ ConnectionID: conn.ConnectionID, diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go new file mode 100644 index 000000000..c2c948a62 --- /dev/null +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -0,0 +1,138 @@ +package challenge + +import ( + "bytes" + "context" + "encoding/json" + "errors" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/zap" + "gorm.io/gorm" + + "github.com/0chain/blobber/code/go/0chain.net/core/logging" +) + +type BCChallengeResponse struct { + BlobberID string `json:"blobber_id"` + Challenges []*ChallengeEntity `json:"challenges"` +} + +// syncChallenges get challenge from blockchain , and add them in database +func syncChallenges(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + logging.Logger.Error("[recover] syncChallenges", zap.Any("err", r)) + } + }() + + params := make(map[string]string) + params["blobber"] = node.Self.ID + + var blobberChallenges BCChallengeResponse + blobberChallenges.Challenges = make([]*ChallengeEntity, 0) + retBytes, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/openchallenges", params, chain.GetServerChain()) + + if err != nil { + logging.Logger.Error("Error getting the open challenges from the blockchain", zap.Error(err)) + } else { + + bytesReader := bytes.NewBuffer(retBytes) + + d := json.NewDecoder(bytesReader) + d.UseNumber() + errd := d.Decode(&blobberChallenges) + + if errd != nil { + logging.Logger.Error("Error in unmarshal of the sharder response", zap.Error(errd)) + } else { + for _, challengeObj := range blobberChallenges.Challenges { + if challengeObj == nil || len(challengeObj.ChallengeID) == 0 { + logging.Logger.Info("No challenge entity from the challenge map") + continue + } + + tx := datastore.GetStore().CreateTransaction(ctx) + db := datastore.GetStore().GetTransaction(tx) + _, err := GetChallengeEntity(tx, challengeObj.ChallengeID) + + // challenge is not synced in db yet + if errors.Is(err, gorm.ErrRecordNotFound) { + + latestChallenge, err := GetLastChallengeEntity(tx) + + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + logging.Logger.Info("Error in load challenge entity from database ", zap.Error(err)) + continue + } + } + + isFirstChallengeInDatabase := len(challengeObj.PrevChallengeID) == 0 || latestChallenge == nil + isNextChallengeOnChain := latestChallenge == nil || latestChallenge.ChallengeID == challengeObj.PrevChallengeID + + if isFirstChallengeInDatabase || isNextChallengeOnChain { + logging.Logger.Info("Adding new challenge found from blockchain", zap.String("challenge", challengeObj.ChallengeID)) + challengeObj.Status = Accepted + if err := challengeObj.Save(tx); err != nil { + logging.Logger.Error("ChallengeEntity_Save", zap.String("challenge_id", challengeObj.ChallengeID), zap.Error(err)) + } + } else { + logging.Logger.Error("Challenge chain is not valid") + } + + } + db.Commit() + tx.Done() + } + } + + } +} + +// processChallenges read and process challenges from db +func processChallenges(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + logging.Logger.Error("[recover] processChallenges", zap.Any("err", r)) + } + }() + rctx := datastore.GetStore().CreateTransaction(ctx) + db := datastore.GetStore().GetTransaction(rctx) + openchallenges := make([]*ChallengeEntity, 0) + db.Where(ChallengeEntity{Status: Accepted}).Find(&openchallenges) + if len(openchallenges) > 0 { + swg := sizedwaitgroup.New(config.Configuration.ChallengeResolveNumWorkers) + for _, openchallenge := range openchallenges { + logging.Logger.Info("Processing the challenge", zap.Any("challenge_id", openchallenge.ChallengeID), zap.Any("openchallenge", openchallenge)) + err := openchallenge.UnmarshalFields() + if err != nil { + logging.Logger.Error("Error unmarshaling challenge entity.", zap.Error(err)) + continue + } + swg.Add() + go func(redeemCtx context.Context, challengeEntity *ChallengeEntity) { + redeemCtx = datastore.GetStore().CreateTransaction(redeemCtx) + defer redeemCtx.Done() + err := LoadValidationTickets(redeemCtx, challengeEntity) + if err != nil { + logging.Logger.Error("Getting validation tickets failed", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) + } + db := datastore.GetStore().GetTransaction(redeemCtx) + err = db.Commit().Error + if err != nil { + logging.Logger.Error("Error commiting the readmarker redeem", zap.Error(err)) + } + swg.Done() + }(ctx, openchallenge) + } + swg.Wait() + } + db.Rollback() + rctx.Done() +} diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 0f2cc5dc5..26b2b2c59 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -3,6 +3,8 @@ package challenge import ( "context" "encoding/json" + "fmt" + "math" "math/rand" "time" @@ -67,7 +69,8 @@ func (cr *ChallengeEntity) ErrorChallenge(ctx context.Context, err error) { } } -func (cr *ChallengeEntity) GetValidationTickets(ctx context.Context) error { +// LoadValidationTickets load validation tickets +func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { if len(cr.Validators) == 0 { cr.StatusMessage = "No validators assigned to the challange" if err := cr.Save(ctx); err != nil { @@ -93,7 +96,6 @@ func (cr *ChallengeEntity) GetValidationTickets(ctx context.Context) error { blockNum := int64(0) if rootRef.NumBlocks > 0 { r := rand.New(rand.NewSource(cr.RandomNumber)) - //rand.Seed(cr.RandomNumber) blockNum = r.Int63n(rootRef.NumBlocks) blockNum = blockNum + 1 } else { @@ -139,9 +141,18 @@ func (cr *ChallengeEntity) GetValidationTickets(ctx context.Context) error { inputData.Name = objectPath.Meta["name"].(string) inputData.Path = objectPath.Meta["path"].(string) inputData.Hash = objectPath.Meta["content_hash"].(string) + inputData.ChunkSize = objectPath.ChunkSize + + maxNumBlocks := 1024 + + // the file is too small, some of 1024 blocks is not filled + if objectPath.Size < objectPath.ChunkSize { + merkleChunkSize := objectPath.ChunkSize / 1024 + maxNumBlocks = int(math.Ceil(float64(objectPath.Size) / float64(merkleChunkSize))) + } + r := rand.New(rand.NewSource(cr.RandomNumber)) - //rand.Seed(cr.RandomNumber) - blockoffset := r.Intn(1024) + blockoffset := r.Intn(maxNumBlocks) blockData, mt, err := filestore.GetFileStore().GetFileBlockForChallenge(cr.AllocationID, inputData, blockoffset) if err != nil { @@ -150,6 +161,7 @@ func (cr *ChallengeEntity) GetValidationTickets(ctx context.Context) error { } postData["data"] = []byte(blockData) postData["merkle_path"] = mt.GetPathByIndex(blockoffset) + postData["chunk_size"] = objectPath.ChunkSize } postDataBytes, err := json.Marshal(postData) @@ -169,7 +181,9 @@ func (cr *ChallengeEntity) GetValidationTickets(ctx context.Context) error { continue } } + url := validator.URL + VALIDATOR_URL + fmt.Println(url) resp, err := util.SendPostRequest(url, postDataBytes, nil) if err != nil { Logger.Info("Got error from the validator.", zap.Any("error", err.Error())) @@ -218,7 +232,7 @@ func (cr *ChallengeEntity) GetValidationTickets(ctx context.Context) error { cr.Result = ChallengeSuccess } else { cr.Result = ChallengeFailure - Logger.Error("Challenge failed by the validators", zap.Any("block_num", cr.BlockNum), zap.Any("object_path", objectPath), zap.Any("challenge", cr)) + //Logger.Error("Challenge failed by the validators", zap.Any("block_num", cr.BlockNum), zap.Any("object_path", objectPath), zap.Any("challenge", cr)) } cr.Status = Processed diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index d33f7a66b..abc23e4e7 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -1,56 +1,52 @@ package challenge import ( - "bytes" "context" - "encoding/json" - "errors" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/lock" - "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - - "github.com/remeh/sizedwaitgroup" - "gorm.io/gorm" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" ) -type BCChallengeResponse struct { - BlobberID string `json:"blobber_id"` - Challenges []*ChallengeEntity `json:"challenges"` -} - +// SetupWorkers start challenge workers func SetupWorkers(ctx context.Context) { - go FindChallenges(ctx) + go startSyncChallenges(ctx) + go startProcessChallenges(ctx) go SubmitProcessedChallenges(ctx) //nolint:errcheck // goroutines } -func GetValidationTickets(ctx context.Context, challengeObj *ChallengeEntity) error { +// LoadValidationTickets load validation tickets for challenge +func LoadValidationTickets(ctx context.Context, challengeObj *ChallengeEntity) error { mutex := lock.GetMutex(challengeObj.TableName(), challengeObj.ChallengeID) mutex.Lock() - err := challengeObj.GetValidationTickets(ctx) + + defer func() { + if r := recover(); r != nil { + Logger.Error("[recover] LoadValidationTickets", zap.Any("err", r)) + } + }() + + err := challengeObj.LoadValidationTickets(ctx) if err != nil { Logger.Error("Error getting the validation tickets", zap.Error(err), zap.String("challenge_id", challengeObj.ChallengeID)) } - mutex.Unlock() + return err } -func SubmitProcessedChallenges(ctx context.Context) error { +func SubmitProcessedChallenges(ctx context.Context) { for { select { case <-ctx.Done(): - return ctx.Err() + return default: rctx := datastore.GetStore().CreateTransaction(ctx) db := datastore.GetStore().GetTransaction(rctx) - //lastChallengeRedeemed := &ChallengeEntity{} + rows, err := db.Table("challenges"). Select("commit_txn_id, sequence"). Where(ChallengeEntity{Status: Committed}). @@ -147,107 +143,29 @@ func SubmitProcessedChallenges(ctx context.Context) error { } time.Sleep(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) } - - return nil //nolint:govet // need more time to verify } -var iterInprogress = false - -func FindChallenges(ctx context.Context) { +func startProcessChallenges(ctx context.Context) { ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) for { select { case <-ctx.Done(): return case <-ticker.C: - if !iterInprogress { - iterInprogress = true - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) - openchallenges := make([]*ChallengeEntity, 0) - db.Where(ChallengeEntity{Status: Accepted}).Find(&openchallenges) - if len(openchallenges) > 0 { - swg := sizedwaitgroup.New(config.Configuration.ChallengeResolveNumWorkers) - for _, openchallenge := range openchallenges { - Logger.Info("Processing the challenge", zap.Any("challenge_id", openchallenge.ChallengeID), zap.Any("openchallenge", openchallenge)) - err := openchallenge.UnmarshalFields() - if err != nil { - Logger.Error("Error unmarshaling challenge entity.", zap.Error(err)) - continue - } - swg.Add() - go func(redeemCtx context.Context, challengeEntity *ChallengeEntity) { - redeemCtx = datastore.GetStore().CreateTransaction(redeemCtx) - defer redeemCtx.Done() - err := GetValidationTickets(redeemCtx, challengeEntity) - if err != nil { - Logger.Error("Getting validation tickets failed", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) - } - db := datastore.GetStore().GetTransaction(redeemCtx) - err = db.Commit().Error - if err != nil { - Logger.Error("Error commiting the readmarker redeem", zap.Error(err)) - } - swg.Done() - }(ctx, openchallenge) - } - swg.Wait() - } - db.Rollback() - rctx.Done() - - params := make(map[string]string) - params["blobber"] = node.Self.ID - - var blobberChallenges BCChallengeResponse - blobberChallenges.Challenges = make([]*ChallengeEntity, 0) - retBytes, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/openchallenges", params, chain.GetServerChain()) - - if err != nil { - Logger.Error("Error getting the open challenges from the blockchain", zap.Error(err)) - } else { - tCtx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(tCtx) - bytesReader := bytes.NewBuffer(retBytes) - - d := json.NewDecoder(bytesReader) - d.UseNumber() - errd := d.Decode(&blobberChallenges) + processChallenges(ctx) + } + } +} - if errd != nil { - Logger.Error("Error in unmarshal of the sharder response", zap.Error(errd)) - } else { - for _, v := range blobberChallenges.Challenges { - if v == nil || len(v.ChallengeID) == 0 { - Logger.Info("No challenge entity from the challenge map") - continue - } - - challengeObj := v - _, err := GetChallengeEntity(tCtx, challengeObj.ChallengeID) - - if errors.Is(err, gorm.ErrRecordNotFound) { - latestChallenge, err := GetLastChallengeEntity(tCtx) - if err == nil || errors.Is(err, gorm.ErrRecordNotFound) { - if (latestChallenge == nil && len(challengeObj.PrevChallengeID) == 0) || latestChallenge.ChallengeID == challengeObj.PrevChallengeID { - Logger.Info("Adding new challenge found from blockchain", zap.String("challenge", v.ChallengeID)) - challengeObj.Status = Accepted - if err := challengeObj.Save(tCtx); err != nil { - Logger.Error("ChallengeEntity_Save", zap.String("challenge_id", challengeObj.ChallengeID), zap.Error(err)) - } - } else { - Logger.Error("Challenge chain is not valid") - } - } - //go stats.AddNewChallengeEvent(challengeObj.AllocationID, challengeObj.ID) - } - } - } - db.Commit() - tCtx.Done() - } - iterInprogress = false - } +// startSyncChallenges +func startSyncChallenges(ctx context.Context) { + ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + syncChallenges(ctx) } } } diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 332e65110..eae61c8b8 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -49,7 +49,13 @@ func SetupConfig(configPath string) { viper.SetEnvKeyReplacer(replacer) viper.AutomaticEnv() viper.SetConfigName("0chain_blobber") - viper.AddConfigPath(configPath) + + if configPath == "" { + viper.AddConfigPath("./config") + } else { + viper.AddConfigPath(configPath) + } + err := viper.ReadInConfig() // Find and read the config file if err != nil { // Handle errors reading the config file panic(fmt.Errorf("fatal error config file: %s", err)) diff --git a/code/go/0chain.net/blobbercore/constants/context_key.go b/code/go/0chain.net/blobbercore/constants/context_key.go deleted file mode 100644 index d7be3edc7..000000000 --- a/code/go/0chain.net/blobbercore/constants/context_key.go +++ /dev/null @@ -1,12 +0,0 @@ -package constants - -import "github.com/0chain/blobber/code/go/0chain.net/core/common" - -const ( - ALLOCATION_CONTEXT_KEY common.ContextKey = "allocation" - CLIENT_CONTEXT_KEY common.ContextKey = "client" - CLIENT_KEY_CONTEXT_KEY common.ContextKey = "client_key" - - // CLIENT_SIGNATURE_HEADER_KEY represents key for context value passed with common.ClientSignatureHeader request header. - CLIENT_SIGNATURE_HEADER_KEY common.ContextKey = "signature" -) diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index 4934027e4..e1855d415 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -427,7 +427,7 @@ func convertDirMetaDataGRPCToDirRef(dirref *blobbergrpc.DirMetaData) *reference. } func WriteFileGRPCToHTTP(req *blobbergrpc.UploadFileRequest) (*http.Request, error) { - var formData allocation.UpdateFileChange + var formData allocation.UpdateFileChanger var uploadMetaString string switch req.Method { case `POST`: diff --git a/code/go/0chain.net/blobbercore/datastore/mock_store.go b/code/go/0chain.net/blobbercore/datastore/mock_store.go index c10457bba..1b0de35eb 100644 --- a/code/go/0chain.net/blobbercore/datastore/mock_store.go +++ b/code/go/0chain.net/blobbercore/datastore/mock_store.go @@ -11,7 +11,6 @@ import ( "gorm.io/gorm" ) - func MocketTheStore(t *testing.T, logging bool) { var err error @@ -27,11 +26,15 @@ func MocketTheStore(t *testing.T, logging bool) { gdb, err := gorm.Open(dialect, new(gorm.Config)) require.NoError(t, err) - setDB(gdb) + //setDB(gdb) + + instance = &postgresStore{ + db: gdb, + } } // sqlmock has problems with inserts, so use mocket for tests with inserts -// https://github.com/DATA-DOG/go-sqlmock/issues/118 + func MockTheStore(t *testing.T) sqlmock.Sqlmock { var db *sql.DB var mock sqlmock.Sqlmock @@ -49,7 +52,9 @@ func MockTheStore(t *testing.T) sqlmock.Sqlmock { gdb, err = gorm.Open(dialector, &gorm.Config{}) require.NoError(t, err) - setDB(gdb) + instance = &postgresStore{ + db: gdb, + } return mock } diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go new file mode 100644 index 000000000..248695263 --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/mocket.go @@ -0,0 +1,88 @@ +package datastore + +import ( + "context" + + . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + mocket "github.com/selvatico/go-mocket" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var mocketInstance *Mocket + +// UseMocket use mocket to mock sql driver +func UseMocket(logging bool) { + if mocketInstance == nil { + mocketInstance = &Mocket{} + mocketInstance.logging = logging + err := mocketInstance.Open() + if err != nil { + panic("UseMocket: " + err.Error()) + } + } + + instance = mocketInstance +} + +// Mocket mock sql driver in data-dog/sqlmock +type Mocket struct { + logging bool + db *gorm.DB +} + +func (store *Mocket) Open() error { + + mocket.Catcher.Reset() + mocket.Catcher.Register() + mocket.Catcher.Logging = store.logging + + dialector := postgres.New(postgres.Config{ + DSN: "mockdb", + DriverName: mocket.DriverName, + PreferSimpleProtocol: true, + }) + + cfg := &gorm.Config{} + + if !store.logging { + cfg.Logger = logger.Default.LogMode(logger.Silent) + } + + gdb, err := gorm.Open(dialector, cfg) + if err != nil { + return err + } + + store.db = gdb + + return nil +} + +func (store *Mocket) Close() { + if store.db != nil { + + if db, _ := store.db.DB(); db != nil { + db.Close() + } + } +} + +func (store *Mocket) CreateTransaction(ctx context.Context) context.Context { + db := store.db.Begin() + return context.WithValue(ctx, ContextKeyTransaction, db) +} + +func (store *Mocket) GetTransaction(ctx context.Context) *gorm.DB { + conn := ctx.Value(ContextKeyTransaction) + if conn != nil { + return conn.(*gorm.DB) + } + Logger.Error("No connection in the context.") + return nil +} + +func (store *Mocket) GetDB() *gorm.DB { + return store.db +} diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go new file mode 100644 index 000000000..98a0db1f0 --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -0,0 +1,68 @@ +package datastore + +import ( + "context" + "fmt" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +// postgresStore store implementation for postgres +type postgresStore struct { + db *gorm.DB +} + +func (store *postgresStore) Open() error { + db, err := gorm.Open(postgres.Open(fmt.Sprintf( + "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", + config.Configuration.DBHost, config.Configuration.DBPort, + config.Configuration.DBUserName, config.Configuration.DBName, + config.Configuration.DBPassword)), &gorm.Config{}) + if err != nil { + return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) + } + + sqldb, err := db.DB() + if err != nil { + return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) + } + + sqldb.SetMaxIdleConns(100) + sqldb.SetMaxOpenConns(200) + sqldb.SetConnMaxLifetime(30 * time.Second) + // Enable Logger, show detailed log + //db.LogMode(true) + store.db = db + return nil +} + +func (store *postgresStore) Close() { + if store.db != nil { + if sqldb, _ := store.db.DB(); sqldb != nil { + sqldb.Close() + } + } +} + +func (store *postgresStore) CreateTransaction(ctx context.Context) context.Context { + db := store.db.Begin() + return context.WithValue(ctx, ContextKeyTransaction, db) +} + +func (store *postgresStore) GetTransaction(ctx context.Context) *gorm.DB { + conn := ctx.Value(ContextKeyTransaction) + if conn != nil { + return conn.(*gorm.DB) + } + Logger.Error("No connection in the context.") + return nil +} + +func (store *postgresStore) GetDB() *gorm.DB { + return store.db +} diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go new file mode 100644 index 000000000..f13fbb365 --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go @@ -0,0 +1,83 @@ +package datastore + +import ( + "context" + + . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/DATA-DOG/go-sqlmock" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var sqlmockInstance *Sqlmock + +// UseSqlmock use sqlmock to mock sql driver +func UseSqlmock() { + if sqlmockInstance == nil { + sqlmockInstance = &Sqlmock{} + err := sqlmockInstance.Open() + if err != nil { + panic("UseSqlmock: " + err.Error()) + } + } + + instance = sqlmockInstance +} + +// Sqlmock mock sql driver in data-dog/sqlmock +type Sqlmock struct { + db *gorm.DB + Sqlmock sqlmock.Sqlmock +} + +func (store *Sqlmock) Open() error { + + db, mock, err := sqlmock.New() + if err != nil { + return err + } + + var dialector = postgres.New(postgres.Config{ + DSN: "sqlmock_db_0", + DriverName: "postgres", + Conn: db, + PreferSimpleProtocol: true, + }) + var gdb *gorm.DB + gdb, err = gorm.Open(dialector, &gorm.Config{}) + if err != nil { + return err + } + + store.db = gdb + store.Sqlmock = mock + + return nil +} + +func (store *Sqlmock) Close() { + if store.db != nil { + + if db, _ := store.db.DB(); db != nil { + db.Close() + } + } +} + +func (store *Sqlmock) CreateTransaction(ctx context.Context) context.Context { + db := store.db.Begin() + return context.WithValue(ctx, ContextKeyTransaction, db) +} + +func (store *Sqlmock) GetTransaction(ctx context.Context) *gorm.DB { + conn := ctx.Value(ContextKeyTransaction) + if conn != nil { + return conn.(*gorm.DB) + } + Logger.Error("No connection in the context.") + return nil +} + +func (store *Sqlmock) GetDB() *gorm.DB { + return store.db +} diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index 529a3016a..bfaae071d 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -2,82 +2,45 @@ package datastore import ( "context" - "fmt" - "time" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "gorm.io/driver/postgres" "gorm.io/gorm" - - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) type contextKey int -const CONNECTION_CONTEXT_KEY contextKey = iota - -type Store struct { - db *gorm.DB -} +const ( + ContextKeyTransaction contextKey = iota + ContextKeyStore +) -var store Store +type Store interface { -func setDB(db *gorm.DB) { - store.db = db -} + // GetDB get raw gorm db + GetDB() *gorm.DB + // CreateTransaction create transaction, and save it in context + CreateTransaction(ctx context.Context) context.Context + // GetTransaction get transaction from context + GetTransaction(ctx context.Context) *gorm.DB -func GetStore() *Store { - return &store + Open() error + Close() } -func (store *Store) Open() error { - db, err := gorm.Open(postgres.Open(fmt.Sprintf( - "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", - config.Configuration.DBHost, config.Configuration.DBPort, - config.Configuration.DBUserName, config.Configuration.DBName, - config.Configuration.DBPassword)), &gorm.Config{}) - if err != nil { - return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) - } +var instance Store - sqldb, err := db.DB() - if err != nil { - return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) - } - - sqldb.SetMaxIdleConns(100) - sqldb.SetMaxOpenConns(200) - sqldb.SetConnMaxLifetime(30 * time.Second) - // Enable Logger, show detailed log - //db.LogMode(true) - store.db = db - return nil -} - -func (store *Store) Close() { - if store.db != nil { - if sqldb, _ := store.db.DB(); sqldb != nil { - sqldb.Close() - } - } +func init() { + instance = &postgresStore{} } -func (store *Store) CreateTransaction(ctx context.Context) context.Context { - db := store.db.Begin() - return context.WithValue(ctx, CONNECTION_CONTEXT_KEY, db) //nolint:staticcheck // changing type might require further refactor +func GetStore() Store { + return instance } -func (store *Store) GetTransaction(ctx context.Context) *gorm.DB { - conn := ctx.Value(CONNECTION_CONTEXT_KEY) - if conn != nil { - return conn.(*gorm.DB) +func FromContext(ctx context.Context) Store { + store := ctx.Value(ContextKeyStore) + if store != nil { + return store.(Store) } - Logger.Error("No connection in the context.") - return nil -} -func (store *Store) GetDB() *gorm.DB { - return store.db + return GetStore() } diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index 728b703f2..f17350927 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -9,13 +9,14 @@ import ( "fmt" "hash" "io" - "io/ioutil" + "math" "mime/multipart" "os" "path/filepath" "strings" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/errors" "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -23,9 +24,11 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/core/util" + "github.com/0chain/gosdk/constants" "github.com/minio/minio-go" "golang.org/x/crypto/sha3" + + "github.com/0chain/gosdk/core/util" ) const ( @@ -82,18 +85,18 @@ func (FileBlockGetter) GetFileBlock(fs *FileFSStore, allocationID string, fileDa return nil, err } - filesize := int(fileinfo.Size()) - maxBlockNum := int64(filesize / CHUNK_SIZE) + filesize := fileinfo.Size() + maxBlockNum := filesize / fileData.ChunkSize // check for any left over bytes. Add one more go routine if required. - if remainder := filesize % CHUNK_SIZE; remainder != 0 { + if remainder := filesize % fileData.ChunkSize; remainder != 0 { maxBlockNum++ } if blockNum > maxBlockNum || blockNum < 1 { return nil, common.NewError("invalid_block_number", "Invalid block number") } - buffer := make([]byte, CHUNK_SIZE*numBlocks) - n, err := file.ReadAt(buffer, ((blockNum - 1) * CHUNK_SIZE)) + buffer := make([]byte, int64(fileData.ChunkSize)*numBlocks) + n, err := file.ReadAt(buffer, ((blockNum - 1) * int64(fileData.ChunkSize))) if err != nil && err != io.EOF { return nil, err } @@ -114,6 +117,16 @@ type StoreAllocation struct { TempObjectsPath string } +var fileFSStore *FileFSStore + +func UseDisk() { + if fileFSStore == nil { + panic("UseDisk: please SetupFSStore first") + } + + fileStore = fileFSStore +} + func SetupFSStore(rootDir string) (FileStore, error) { if err := createDirs(rootDir); err != nil { return nil, err @@ -122,12 +135,15 @@ func SetupFSStore(rootDir string) (FileStore, error) { } func SetupFSStoreI(rootDir string, fileBlockGetter IFileBlockGetter) (FileStore, error) { - fsStore = &FileFSStore{ + fileFSStore = &FileFSStore{ RootDirectory: rootDir, Minio: intializeMinio(), fileBlockGetter: fileBlockGetter, } - return fsStore, nil + + fileStore = fileFSStore + + return fileStore, nil } func intializeMinio() *minio.Client { @@ -300,49 +316,51 @@ func (fs *FileFSStore) GetFileBlockForChallenge(allocationID string, fileData *F var returnBytes []byte - merkleHashes := make([]hash.Hash, 1024) - merkleLeaves := make([]util.Hashable, 1024) - for idx := range merkleHashes { - merkleHashes[idx] = sha3.New256() - } + fi, _ := file.Stat() + + numChunks := int(math.Ceil(float64(fi.Size()) / float64(fileData.ChunkSize))) + + fmt := util.NewFixedMerkleTree(int(fileData.ChunkSize)) + bytesBuf := bytes.NewBuffer(make([]byte, 0)) - for { - _, err := io.CopyN(bytesBuf, file, CHUNK_SIZE) - if err != io.EOF && err != nil { - return nil, nil, common.NewError("file_write_error", err.Error()) - } - dataBytes := bytesBuf.Bytes() - tmpBytes := make([]byte, len(dataBytes)) - copy(tmpBytes, dataBytes) - merkleChunkSize := 64 - for i := 0; i < len(tmpBytes); i += merkleChunkSize { - end := i + merkleChunkSize - if end > len(tmpBytes) { - end = len(tmpBytes) + for chunkIndex := 0; chunkIndex < numChunks; chunkIndex++ { + written, err := io.CopyN(bytesBuf, file, fileData.ChunkSize) + + if written > 0 { + dataBytes := bytesBuf.Bytes() + + err2 := fmt.Write(dataBytes, chunkIndex) + if err2 != nil { + return nil, nil, errors.ThrowLog(err2.Error(), constants.ErrUnableHash) } - offset := i / merkleChunkSize - merkleHashes[offset].Write(tmpBytes[i:end]) - if offset == blockoffset { - returnBytes = append(returnBytes, tmpBytes[i:end]...) + + merkleChunkSize := int(fileData.ChunkSize / 1024) + for i := 0; i < len(dataBytes); i += merkleChunkSize { + end := i + merkleChunkSize + if end > len(dataBytes) { + end = len(dataBytes) + } + offset := i / merkleChunkSize + + if offset == blockoffset { + returnBytes = append(returnBytes, dataBytes[i:end]...) + } } + bytesBuf.Reset() } - bytesBuf.Reset() + if err != nil && err == io.EOF { break } } - for idx := range merkleHashes { - merkleLeaves[idx] = util.NewStringHashable(hex.EncodeToString(merkleHashes[idx].Sum(nil))) - } - var mt util.MerkleTreeI = &util.MerkleTree{} - mt.ComputeTree(merkleLeaves) - - return returnBytes, mt, nil + return returnBytes, fmt.GetMerkleTree(), nil } func (fs *FileFSStore) GetFileBlock(allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { + return fs.fileBlockGetter.GetFileBlock(fs, allocationID, fileData, blockNum, numBlocks) + } func (fs *FileFSStore) DeleteTempFile(allocationID string, fileData *FileInputData, connectionID string) error { @@ -426,72 +444,6 @@ func (fs *FileFSStore) DeleteFile(allocationID string, contentHash string) error return os.Remove(fileObjectPath) } -func (fs *FileFSStore) GetMerkleTreeForFile(allocationID string, fileData *FileInputData) (util.MerkleTreeI, error) { - allocation, err := fs.SetupAllocation(allocationID, true) - if err != nil { - return nil, common.NewError("filestore_setup_error", "Error setting the fs store. "+err.Error()) - } - dirPath, destFile := GetFilePathFromHash(fileData.Hash) - fileObjectPath := filepath.Join(allocation.ObjectsPath, dirPath) - fileObjectPath = filepath.Join(fileObjectPath, destFile) - - file, err := os.Open(fileObjectPath) - if err != nil { - if os.IsNotExist(err) && fileData.OnCloud { - err = fs.DownloadFromCloud(fileData.Hash, fileObjectPath) - if err != nil { - return nil, common.NewError("minio_download_failed", "Unable to download from minio with err "+err.Error()) - } - file, err = os.Open(fileObjectPath) - if err != nil { - return nil, err - } - } else { - return nil, err - } - } - defer file.Close() - //merkleHash := sha3.New256() - tReader := file //io.TeeReader(file, merkleHash) - //merkleLeaves := make([]util.Hashable, 0) - merkleHashes := make([]hash.Hash, 1024) - merkleLeaves := make([]util.Hashable, 1024) - for idx := range merkleHashes { - merkleHashes[idx] = sha3.New256() - } - bytesBuf := bytes.NewBuffer(make([]byte, 0)) - for { - _, err := io.CopyN(bytesBuf, tReader, CHUNK_SIZE) - if err != io.EOF && err != nil { - return nil, common.NewError("file_write_error", err.Error()) - } - dataBytes := bytesBuf.Bytes() - merkleChunkSize := 64 - for i := 0; i < len(dataBytes); i += merkleChunkSize { - end := i + merkleChunkSize - if end > len(dataBytes) { - end = len(dataBytes) - } - offset := i / merkleChunkSize - merkleHashes[offset].Write(dataBytes[i:end]) - } - //merkleLeaves = append(merkleLeaves, util.NewStringHashable(hex.EncodeToString(merkleHash.Sum(nil)))) - //merkleHash.Reset() - bytesBuf.Reset() - if err != nil && err == io.EOF { - break - } - } - for idx := range merkleHashes { - merkleLeaves[idx] = util.NewStringHashable(hex.EncodeToString(merkleHashes[idx].Sum(nil))) - } - - var mt util.MerkleTreeI = &util.MerkleTree{} - mt.ComputeTree(merkleLeaves) - - return mt, nil -} - func (fs *FileFSStore) CreateDir(dirName string) error { return createDirs(dirName) } @@ -503,48 +455,28 @@ func (fs *FileFSStore) DeleteDir(allocationID, dirPath, connectionID string) err func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, infile multipart.File, connectionID string) (*FileOutputData, error) { + if fileData.IsChunked { + return fs.WriteChunk(allocationID, fileData, infile, connectionID) + } + allocation, err := fs.SetupAllocation(allocationID, false) if err != nil { return nil, common.NewError("filestore_setup_error", "Error setting the fs store. "+err.Error()) } tempFilePath := fs.generateTempPath(allocation, fileData, connectionID) - dest, err := NewChunkWriter(tempFilePath) + dest, err := os.Create(tempFilePath) if err != nil { return nil, common.NewError("file_creation_error", err.Error()) } defer dest.Close() fileRef := &FileOutputData{} - var fileReader io.Reader = infile - - if fileData.IsResumable { - h := sha1.New() - offset, err := dest.WriteChunk(context.TODO(), fileData.UploadOffset, io.TeeReader(fileReader, h)) - - if err != nil { - return nil, common.NewError("file_write_error", err.Error()) - } - - fileRef.ContentHash = hex.EncodeToString(h.Sum(nil)) - fileRef.Size = dest.Size() - fileRef.Name = fileData.Name - fileRef.Path = fileData.Path - fileRef.UploadOffset = fileData.UploadOffset + offset - fileRef.UploadLength = fileData.UploadLength - - if !fileData.IsFinal { - //skip to compute hash until the last chunk is uploaded - return fileRef, nil - } - - fileReader = dest - } h := sha1.New() bytesBuffer := bytes.NewBuffer(nil) multiHashWriter := io.MultiWriter(h, bytesBuffer) - tReader := io.TeeReader(fileReader, multiHashWriter) + tReader := io.TeeReader(infile, multiHashWriter) merkleHashes := make([]hash.Hash, 1024) merkleLeaves := make([]util.Hashable, 1024) for idx := range merkleHashes { @@ -552,14 +484,8 @@ func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, } fileSize := int64(0) for { - var written int64 - if fileData.IsResumable { - //all chunks have been written, only read bytes from local file , and compute hash - written, err = io.CopyN(ioutil.Discard, tReader, CHUNK_SIZE) - } else { - written, err = io.CopyN(dest, tReader, CHUNK_SIZE) - } + written, err := io.CopyN(dest, tReader, CHUNK_SIZE) if err != io.EOF && err != nil { return nil, common.NewError("file_write_error", err.Error()) @@ -588,17 +514,50 @@ func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, var mt util.MerkleTreeI = &util.MerkleTree{} mt.ComputeTree(merkleLeaves) - //only update hash for whole file when it is not a resumable upload or is final chunk. - if !fileData.IsResumable || fileData.IsFinal { - fileRef.ContentHash = hex.EncodeToString(h.Sum(nil)) - } - + fileRef.ContentHash = hex.EncodeToString(h.Sum(nil)) fileRef.Size = fileSize fileRef.Name = fileData.Name fileRef.Path = fileData.Path fileRef.MerkleRoot = mt.GetRoot() - fileRef.UploadOffset = fileSize - fileRef.UploadLength = fileData.UploadLength + + return fileRef, nil +} + +// WriteChunk append chunk to temp file +func (fs *FileFSStore) WriteChunk(allocationID string, fileData *FileInputData, + infile multipart.File, connectionID string) (*FileOutputData, error) { + + allocation, err := fs.SetupAllocation(allocationID, false) + if err != nil { + return nil, common.NewError("filestore_setup_error", "Error setting the fs store. "+err.Error()) + } + + tempFilePath := fs.generateTempPath(allocation, fileData, connectionID) + dest, err := NewChunkWriter(tempFilePath) + if err != nil { + return nil, common.NewError("file_creation_error", err.Error()) + } + defer dest.Close() + + fileRef := &FileOutputData{} + + // the chunk has been rewitten. but it is lost when network is broken, and it is not save in db + if dest.size > fileData.UploadOffset { + fileRef.ChunkUploaded = true + } + + h := sha1.New() + size, err := dest.WriteChunk(context.TODO(), fileData.UploadOffset, io.TeeReader(infile, h)) + + if err != nil { + return nil, errors.ThrowLog(err.Error(), constants.ErrUnableWriteFile) + } + + fileRef.Size = size + fileRef.ContentHash = hex.EncodeToString(h.Sum(nil)) + + fileRef.Name = fileData.Name + fileRef.Path = fileData.Path return fileRef, nil } diff --git a/code/go/0chain.net/blobbercore/filestore/mock_store.go b/code/go/0chain.net/blobbercore/filestore/mock_store.go new file mode 100644 index 000000000..fcda9fc75 --- /dev/null +++ b/code/go/0chain.net/blobbercore/filestore/mock_store.go @@ -0,0 +1,101 @@ +package filestore + +import ( + "crypto/sha1" + "encoding/hex" + "encoding/json" + "io" + "mime/multipart" + + "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk/core/util" +) + +type MockStore struct { +} + +var mockStore *MockStore + +func UseMock() { + if mockStore == nil { + mockStore = &MockStore{} + + } + + fileStore = mockStore +} + +// WriteFile write chunk file into disk +func (ms *MockStore) WriteFile(allocationID string, fileData *FileInputData, infile multipart.File, connectionID string) (*FileOutputData, error) { + fileRef := &FileOutputData{} + + fileRef.ChunkUploaded = true + + h := sha1.New() + reader := io.TeeReader(infile, h) + fileSize := int64(0) + for { + + written, err := io.CopyN(io.Discard, reader, fileData.ChunkSize) + + fileSize += written + + if err != nil { + break + } + } + + fileRef.Size = fileSize + fileRef.ContentHash = hex.EncodeToString(h.Sum(nil)) + + fileRef.Name = fileData.Name + fileRef.Path = fileData.Path + + return fileRef, nil +} +func (ms *MockStore) DeleteTempFile(allocationID string, fileData *FileInputData, connectionID string) error { + return nil +} + +func (ms *MockStore) CreateDir(dirName string) error { + return nil +} +func (ms *MockStore) DeleteDir(allocationID, dirPath, connectionID string) error { + return nil +} + +func (ms *MockStore) GetFileBlock(allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { + return nil, constants.ErrNotImplemented +} + +func (ms *MockStore) CommitWrite(allocationID string, fileData *FileInputData, connectionID string) (bool, error) { + return true, nil +} + +func (ms *MockStore) GetFileBlockForChallenge(allocationID string, fileData *FileInputData, blockoffset int) (json.RawMessage, util.MerkleTreeI, error) { + return nil, nil, constants.ErrNotImplemented +} +func (ms *MockStore) DeleteFile(allocationID string, contentHash string) error { + return nil +} +func (ms *MockStore) GetTotalDiskSizeUsed() (int64, error) { + return 0, constants.ErrNotImplemented +} +func (ms *MockStore) GetlDiskSizeUsed(allocationID string) (int64, error) { + return 0, constants.ErrNotImplemented +} +func (ms *MockStore) GetTempPathSize(allocationID string) (int64, error) { + return 0, constants.ErrNotImplemented +} +func (ms *MockStore) IterateObjects(allocationID string, handler FileObjectHandler) error { + return nil +} +func (ms *MockStore) UploadToCloud(fileHash, filePath string) error { + return nil +} +func (ms *MockStore) DownloadFromCloud(fileHash, filePath string) error { + return nil +} +func (ms *MockStore) SetupAllocation(allocationID string, skipCreate bool) (*StoreAllocation, error) { + return nil, constants.ErrNotImplemented +} diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index 565b27d15..305ad2165 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -4,7 +4,7 @@ import ( "encoding/json" "mime/multipart" - "github.com/0chain/blobber/code/go/0chain.net/core/util" + "github.com/0chain/gosdk/core/util" ) const CHUNK_SIZE = 64 * 1024 @@ -15,8 +15,10 @@ type FileInputData struct { Hash string OnCloud bool - //IsResumable the request is resumable upload - IsResumable bool + // ChunkSize chunk size + ChunkSize int64 + //IsChunked the request is chunked upload + IsChunked bool //UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. UploadLength int64 //Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. @@ -30,24 +32,26 @@ type FileOutputData struct { Path string MerkleRoot string ContentHash string - Size int64 - - //UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. - UploadLength int64 - //Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. - UploadOffset int64 + // Size wirtten size/chunk size + Size int64 + // ChunkUploaded the chunk is uploaded or not. + ChunkUploaded bool } type FileObjectHandler func(contentHash string, contentSize int64) type FileStore interface { + // WriteFile write chunk file into disk WriteFile(allocationID string, fileData *FileInputData, infile multipart.File, connectionID string) (*FileOutputData, error) DeleteTempFile(allocationID string, fileData *FileInputData, connectionID string) error + CreateDir(dirName string) error DeleteDir(allocationID, dirPath, connectionID string) error + GetFileBlock(allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) + CommitWrite(allocationID string, fileData *FileInputData, connectionID string) (bool, error) - //GetMerkleTreeForFile(allocationID string, fileData *FileInputData) (util.MerkleTreeI, error) + GetFileBlockForChallenge(allocationID string, fileData *FileInputData, blockoffset int) (json.RawMessage, util.MerkleTreeI, error) DeleteFile(allocationID string, contentHash string) error GetTotalDiskSizeUsed() (int64, error) @@ -59,8 +63,8 @@ type FileStore interface { SetupAllocation(allocationID string, skipCreate bool) (*StoreAllocation, error) } -var fsStore FileStore +var fileStore FileStore func GetFileStore() FileStore { - return fsStore + return fileStore } diff --git a/code/go/0chain.net/blobbercore/handler/commit_integration_test.go b/code/go/0chain.net/blobbercore/handler/commit_integration_test.go index 855fd0cbb..479f6f6be 100644 --- a/code/go/0chain.net/blobbercore/handler/commit_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/commit_integration_test.go @@ -4,11 +4,12 @@ import ( "context" "encoding/hex" "encoding/json" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "strconv" "testing" "time" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -37,6 +38,7 @@ func TestBlobberGRPCService_Commit(t *testing.T) { ContentHash: "contentHash", MerkleRoot: "merkleRoot", ActualFileHash: "actualFileHash", + ChunkSize: 65536, } rootRefHash := encryption.Hash(encryption.Hash(fr.GetFileHashData())) diff --git a/code/go/0chain.net/blobbercore/handler/file_command.go b/code/go/0chain.net/blobbercore/handler/file_command.go new file mode 100644 index 000000000..9601b5248 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/file_command.go @@ -0,0 +1,39 @@ +package handler + +import ( + "context" + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" +) + +// FileCommand execute command for a file operation +type FileCommand interface { + // IsAuthorized validate request, and try build ChangeProcesser instance + IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error + + // ProcessContent flush file to FileStorage + ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) + + // ProcessThumbnail flush thumbnail file to FileStorage if it has. + ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error + + // UpdateChange update AllocationChangeProcessor. It will be president in db for commiting transcation + UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error +} + +// createFileCommand create file command for UPLOAD,UPDATE and DELETE +func createFileCommand(req *http.Request) FileCommand { + switch req.Method { + case http.MethodPost: + return &AddFileCommand{} + case http.MethodPut: + return &UpdateFileCommand{} + case http.MethodDelete: + return &FileCommandDelete{} + + default: + return &AddFileCommand{} + } +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_add.go b/code/go/0chain.net/blobbercore/handler/file_command_add.go new file mode 100644 index 000000000..edcf57aa0 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/file_command_add.go @@ -0,0 +1,191 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk/zboxcore/fileref" + "go.uber.org/zap" +) + +// AddFileCommand command for resuming file +type AddFileCommand struct { + allocationChange *allocation.AllocationChange + fileChanger *allocation.AddFileChanger +} + +// IsAuthorized validate request. +func (cmd *AddFileCommand) IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { + if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { + return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") + } + + fileChanger := &allocation.AddFileChanger{} + + uploadMetaString := req.FormValue("uploadMeta") + err := json.Unmarshal([]byte(uploadMetaString), fileChanger) + if err != nil { + return common.NewError("invalid_parameters", + "Invalid parameters. Error parsing the meta data for upload."+err.Error()) + } + exisitingFileRef, _ := reference.GetReference(ctx, allocationObj.ID, fileChanger.Path) + + if exisitingFileRef != nil { + return common.NewError("duplicate_file", "File at path already exists") + } + + //create a FixedMerkleTree instance first, it will be reloaded from db in cmd.reloadChange if it is not first chunk + //cmd.fileChanger.FixedMerkleTree = &util.FixedMerkleTree{} + + if fileChanger.ChunkSize <= 0 { + fileChanger.ChunkSize = fileref.CHUNK_SIZE + } + + cmd.fileChanger = fileChanger + + return nil + +} + +// ProcessContent flush file to FileStorage +func (cmd *AddFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { + result := blobberhttp.UploadResult{} + + origfile, _, err := req.FormFile("uploadFile") + if err != nil { + return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) + } + defer origfile.Close() + + cmd.reloadChange(connectionObj) + + fileInputData := &filestore.FileInputData{ + Name: cmd.fileChanger.Filename, + Path: cmd.fileChanger.Path, + OnCloud: false, + + ChunkSize: cmd.fileChanger.ChunkSize, + UploadOffset: cmd.fileChanger.UploadOffset, + IsChunked: true, + IsFinal: cmd.fileChanger.IsFinal, + } + fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) + if err != nil { + return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) + } + + result.Filename = cmd.fileChanger.Filename + result.Hash = fileOutputData.ContentHash + //result.MerkleRoot = fileOutputData.MerkleRoot + result.Size = fileOutputData.Size + + allocationSize := connectionObj.Size + + // only update connection size when the chunk is uploaded by first time. + if !fileOutputData.ChunkUploaded { + allocationSize += fileOutputData.Size + } + + if allocationSize > config.Configuration.MaxFileSize { + return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") + } + + if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize { + return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") + } + + if len(cmd.fileChanger.ChunkHash) > 0 && cmd.fileChanger.ChunkHash != fileOutputData.ContentHash { + return result, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") + } + + // Save client's ContentHash in database instead blobber's + // it saves time to read and compute hash of fragment from disk again + //cmd.fileChanger.Hash = fileOutputData.ContentHash + + cmd.fileChanger.AllocationID = allocationObj.ID + cmd.fileChanger.Size = allocationSize + + cmd.allocationChange = &allocation.AllocationChange{} + cmd.allocationChange.ConnectionID = connectionObj.ConnectionID + cmd.allocationChange.Size = allocationSize + cmd.allocationChange.Operation = constants.FileOperationInsert + + connectionObj.Size = allocationSize + + return result, nil +} + +// ProcessThumbnail flush thumbnail file to FileStorage if it has. +func (cmd *AddFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { + + thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") + + if thumbHeader != nil { + + defer thumbfile.Close() + + thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} + thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, thumbInputData, thumbfile, connectionObj.ConnectionID) + if err != nil { + return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) + } + if cmd.fileChanger.ThumbnailHash != thumbOutputData.ContentHash { + return common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") + } + + cmd.fileChanger.ThumbnailHash = thumbOutputData.ContentHash + cmd.fileChanger.ThumbnailSize = thumbOutputData.Size + cmd.fileChanger.ThumbnailFilename = thumbInputData.Name + } + + return nil +} + +func (cmd *AddFileCommand) reloadChange(connectionObj *allocation.AllocationChangeCollector) { + for _, c := range connectionObj.Changes { + if c.Operation == constants.FileOperationInsert { + + dbChangeProcessor := &allocation.AddFileChanger{} + + err := dbChangeProcessor.Unmarshal(c.Input) + if err != nil { + logging.Logger.Error("reloadChange", zap.Error(err)) + } + + cmd.fileChanger.Size = dbChangeProcessor.Size + return + } + } +} + +// UpdateChange replace AddFileChange in db +func (cmd *AddFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { + for _, c := range connectionObj.Changes { + if c.Operation == constants.FileOperationInsert { + c.Size = connectionObj.Size + c.Input, _ = cmd.fileChanger.Marshal() + + //c.ModelWithTS.UpdatedAt = time.Now() + err := connectionObj.Save(ctx) + if err != nil { + return err + } + + return c.Save(ctx) + } + } + + //NOT FOUND + connectionObj.AddChange(cmd.allocationChange, cmd.fileChanger) + + return connectionObj.Save(ctx) +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go new file mode 100644 index 000000000..f33ae7014 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -0,0 +1,77 @@ +package handler + +import ( + "context" + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/gosdk/constants" +) + +// FileCommandDelete command for deleting file +type FileCommandDelete struct { + exisitingFileRef *reference.Ref + changeProcessor *allocation.DeleteFileChange + allocationChange *allocation.AllocationChange +} + +// IsAuthorized validate request. +func (cmd *FileCommandDelete) IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { + if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { + return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") + } + + path := req.FormValue("path") + if len(path) == 0 { + return common.NewError("invalid_parameters", "Invalid path") + } + cmd.exisitingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, path) + + if cmd.exisitingFileRef == nil { + return common.NewError("invalid_file", "File does not exist at path") + } + + return nil +} + +// UpdateChange add DeleteFileChange in db +func (cmd *FileCommandDelete) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { + connectionObj.AddChange(cmd.allocationChange, cmd.changeProcessor) + + return connectionObj.Save(ctx) +} + +// ProcessContent flush file to FileStorage +func (cmd *FileCommandDelete) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { + + deleteSize := cmd.exisitingFileRef.Size + + cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionObj.ConnectionID, + AllocationID: connectionObj.AllocationID, Name: cmd.exisitingFileRef.Name, + Hash: cmd.exisitingFileRef.Hash, Path: cmd.exisitingFileRef.Path, Size: deleteSize} + + result := blobberhttp.UploadResult{} + result.Filename = cmd.exisitingFileRef.Name + result.Hash = cmd.exisitingFileRef.Hash + result.MerkleRoot = cmd.exisitingFileRef.MerkleRoot + result.Size = cmd.exisitingFileRef.Size + + cmd.allocationChange = &allocation.AllocationChange{} + cmd.allocationChange.ConnectionID = connectionObj.ConnectionID + cmd.allocationChange.Size = 0 - deleteSize + cmd.allocationChange.Operation = constants.FileOperationDelete + + connectionObj.Size += cmd.allocationChange.Size + + return result, nil + +} + +// ProcessThumbnail no thumbnail should be processed for delete. A deffered delete command has been added on ProcessContent +func (cmd *FileCommandDelete) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { + //DO NOTHING + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_insert.go b/code/go/0chain.net/blobbercore/handler/file_command_insert.go new file mode 100644 index 000000000..81e5de259 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/file_command_insert.go @@ -0,0 +1,139 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk/zboxcore/fileref" +) + +// InsertFileCommand command for inserting file +type InsertFileCommand struct { + allocationChange *allocation.AllocationChange + changeProcessor *allocation.UpdateFileChanger +} + +// IsAuthorized validate request. +func (cmd *InsertFileCommand) IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { + + if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { + return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") + } + + changeProcessor := &allocation.UpdateFileChanger{} + + uploadMetaString := req.FormValue("uploadMeta") + err := json.Unmarshal([]byte(uploadMetaString), changeProcessor) + if err != nil { + return common.NewError("invalid_parameters", + "Invalid parameters. Error parsing the meta data for upload."+err.Error()) + } + exisitingFileRef, _ := reference.GetReference(ctx, allocationObj.ID, changeProcessor.Path) + + if exisitingFileRef != nil { + return common.NewError("duplicate_file", "File at path already exists") + } + + if changeProcessor.ChunkSize <= 0 { + changeProcessor.ChunkSize = fileref.CHUNK_SIZE + } + + cmd.changeProcessor = changeProcessor + + return nil +} + +// ProcessContent flush file to FileStorage +func (cmd *InsertFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { + + result := blobberhttp.UploadResult{} + + origfile, _, err := req.FormFile("uploadFile") + if err != nil { + return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) + } + defer origfile.Close() + + fileInputData := &filestore.FileInputData{Name: cmd.changeProcessor.Filename, Path: cmd.changeProcessor.Path, OnCloud: false} + fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) + if err != nil { + return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) + } + + result.Filename = cmd.changeProcessor.Filename + result.Hash = fileOutputData.ContentHash + result.MerkleRoot = fileOutputData.MerkleRoot + result.Size = fileOutputData.Size + + if len(cmd.changeProcessor.Hash) > 0 && cmd.changeProcessor.Hash != fileOutputData.ContentHash { + return result, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") + } + if len(cmd.changeProcessor.MerkleRoot) > 0 && cmd.changeProcessor.MerkleRoot != fileOutputData.MerkleRoot { + return result, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") + } + if fileOutputData.Size > config.Configuration.MaxFileSize { + return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") + } + + cmd.changeProcessor.Hash = fileOutputData.ContentHash + cmd.changeProcessor.MerkleRoot = fileOutputData.MerkleRoot + cmd.changeProcessor.AllocationID = allocationObj.ID + cmd.changeProcessor.Size = fileOutputData.Size + + allocationSize := fileOutputData.Size + + if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize { + return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") + } + + cmd.allocationChange = &allocation.AllocationChange{} + cmd.allocationChange.ConnectionID = connectionObj.ConnectionID + cmd.allocationChange.Size = allocationSize + cmd.allocationChange.Operation = constants.FileOperationInsert + + connectionObj.Size += cmd.allocationChange.Size + + return result, nil + +} + +// ProcessThumbnail flush thumbnail file to FileStorage if it has. +func (cmd *InsertFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { + + thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") + + if thumbHeader != nil { + + defer thumbfile.Close() + + thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.changeProcessor.Path} + thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, thumbInputData, thumbfile, connectionObj.ConnectionID) + if err != nil { + return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) + } + if len(cmd.changeProcessor.ThumbnailHash) > 0 && cmd.changeProcessor.ThumbnailHash != thumbOutputData.ContentHash { + return common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") + } + + cmd.changeProcessor.ThumbnailHash = thumbOutputData.ContentHash + cmd.changeProcessor.ThumbnailSize = thumbOutputData.Size + cmd.changeProcessor.ThumbnailFilename = thumbInputData.Name + } + + return nil + +} + +// UpdateChange add NewFileChange in db +func (cmd *InsertFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { + connectionObj.AddChange(cmd.allocationChange, cmd.changeProcessor) + return connectionObj.Save(ctx) +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go new file mode 100644 index 000000000..fbae2b6f6 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -0,0 +1,200 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/constants" + sdkConstants "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk/zboxcore/fileref" + "go.uber.org/zap" +) + +// UpdateFileCommand command for updating file +type UpdateFileCommand struct { + exisitingFileRef *reference.Ref + fileChanger *allocation.UpdateFileChanger + allocationChange *allocation.AllocationChange +} + +// IsAuthorized validate request. +func (cmd *UpdateFileCommand) IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { + uploadMetaString := req.FormValue("uploadMeta") + + if uploadMetaString == "" { + // backward compatibility for old update request + uploadMetaString = req.FormValue("updatedMeta") + } + + err := json.Unmarshal([]byte(uploadMetaString), &cmd.fileChanger) + if err != nil { + return common.NewError("invalid_parameters", + "Invalid parameters. Error parsing the meta data for upload."+err.Error()) + } + + if cmd.fileChanger.ChunkSize <= 0 { + cmd.fileChanger.ChunkSize = fileref.CHUNK_SIZE + } + + cmd.exisitingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.fileChanger.Path) + + if cmd.exisitingFileRef == nil { + return common.NewError("invalid_file_update", "File at path does not exist for update") + } + + if allocationObj.OwnerID != clientID && + allocationObj.RepairerID != clientID && + !reference.IsACollaborator(ctx, cmd.exisitingFileRef.ID, clientID) { + return common.NewError("invalid_operation", "Operation needs to be performed by the owner, collaborator or the payer of the allocation") + } + + return nil +} + +// ProcessContent flush file to FileStorage +func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { + + result := blobberhttp.UploadResult{} + + origfile, _, err := req.FormFile("uploadFile") + if err != nil { + return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) + } + defer origfile.Close() + + cmd.reloadChange(connectionObj) + + fileInputData := &filestore.FileInputData{ + Name: cmd.fileChanger.Filename, + Path: cmd.fileChanger.Path, + OnCloud: cmd.exisitingFileRef.OnCloud, + + UploadOffset: cmd.fileChanger.UploadOffset, + IsChunked: cmd.fileChanger.ChunkSize > 0, + IsFinal: cmd.fileChanger.IsFinal, + } + fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) + if err != nil { + return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) + } + + result.Filename = cmd.fileChanger.Filename + result.Hash = fileOutputData.ContentHash + //result.MerkleRoot = fileOutputData.MerkleRoot + result.Size = fileOutputData.Size + + allocationSize := connectionObj.Size + + // only update connection size when the chunk is uploaded by first time. + if !fileOutputData.ChunkUploaded { + allocationSize += fileOutputData.Size + } + + if len(cmd.fileChanger.ChunkHash) > 0 && cmd.fileChanger.ChunkHash != fileOutputData.ContentHash { + return result, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") + } + + // if len(cmd.fileChanger.MerkleRoot) > 0 && cmd.fileChanger.MerkleRoot != fileOutputData.MerkleRoot { + // return result, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") + // } + + if allocationObj.BlobberSizeUsed+(allocationSize-cmd.exisitingFileRef.Size) > allocationObj.BlobberSize { + return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") + } + + if fileOutputData.Size > config.Configuration.MaxFileSize { + return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") + } + + cmd.fileChanger.AllocationID = allocationObj.ID + cmd.fileChanger.Size = allocationSize + + cmd.allocationChange = &allocation.AllocationChange{} + cmd.allocationChange.ConnectionID = connectionObj.ConnectionID + cmd.allocationChange.Size = allocationSize - cmd.exisitingFileRef.Size + cmd.allocationChange.Operation = sdkConstants.FileOperationUpdate + + if cmd.fileChanger.IsFinal { + connectionObj.Size = allocationSize - cmd.exisitingFileRef.Size + } else { + connectionObj.Size = allocationSize + } + + return result, nil + +} + +// ProcessThumbnail flush thumbnail file to FileStorage if it has. +func (cmd *UpdateFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { + + thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") + + if thumbHeader != nil { + + defer thumbfile.Close() + + thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} + thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, thumbInputData, thumbfile, connectionObj.ConnectionID) + if err != nil { + return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) + } + if len(cmd.fileChanger.ThumbnailHash) > 0 && cmd.fileChanger.ThumbnailHash != thumbOutputData.ContentHash { + return common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") + } + cmd.fileChanger.ThumbnailHash = thumbOutputData.ContentHash + cmd.fileChanger.ThumbnailSize = thumbOutputData.Size + cmd.fileChanger.ThumbnailFilename = thumbInputData.Name + } + + return nil + +} + +func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationChangeCollector) { + for _, c := range connectionObj.Changes { + if c.Operation == constants.FileOperationUpdate { + + dbFileChanger := &allocation.AddFileChanger{} + + err := dbFileChanger.Unmarshal(c.Input) + if err != nil { + logging.Logger.Error("reloadChange", zap.Error(err)) + } + + // reload uploaded size from db, it was chunk size from client + cmd.fileChanger.Size = dbFileChanger.Size + return + } + } +} + +// UpdateChange add UpdateFileChanger in db +func (cmd *UpdateFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { + for _, c := range connectionObj.Changes { + if c.Operation == constants.FileOperationUpdate { + c.Size = connectionObj.Size + c.Input, _ = cmd.fileChanger.Marshal() + + //c.ModelWithTS.UpdatedAt = time.Now() + err := connectionObj.Save(ctx) + if err != nil { + return err + } + + return c.Save(ctx) + } + } + + //NOT FOUND + connectionObj.AddChange(cmd.allocationChange, cmd.fileChanger) + + return connectionObj.Save(ctx) +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update_bak.go b/code/go/0chain.net/blobbercore/handler/file_command_update_bak.go new file mode 100644 index 000000000..6bc66f690 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/file_command_update_bak.go @@ -0,0 +1,138 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk/zboxcore/fileref" +) + +// UpdateFileCMD command for updating file +type UpdateFileCMD struct { + exisitingFileRef *reference.Ref + changeProcessor *allocation.UpdateFileChanger + allocationChange *allocation.AllocationChange +} + +// IsAuthorized validate request. +func (cmd *UpdateFileCMD) IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { + uploadMetaString := req.FormValue("updateMeta") + err := json.Unmarshal([]byte(uploadMetaString), &cmd.changeProcessor) + if err != nil { + return common.NewError("invalid_parameters", + "Invalid parameters. Error parsing the meta data for upload."+err.Error()) + } + + if cmd.changeProcessor.ChunkSize <= 0 { + cmd.changeProcessor.ChunkSize = fileref.CHUNK_SIZE + } + + cmd.exisitingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.changeProcessor.Path) + + if cmd.exisitingFileRef == nil { + return common.NewError("invalid_file_update", "File at path does not exist for update") + } + + if allocationObj.OwnerID != clientID && + allocationObj.RepairerID != clientID && + !reference.IsACollaborator(ctx, cmd.exisitingFileRef.ID, clientID) { + return common.NewError("invalid_operation", "Operation needs to be performed by the owner, collaborator or the payer of the allocation") + } + + return nil +} + +// ProcessContent flush file to FileStorage +func (cmd *UpdateFileCMD) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { + + result := blobberhttp.UploadResult{} + + origfile, _, err := req.FormFile("uploadFile") + if err != nil { + return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) + } + defer origfile.Close() + + fileInputData := &filestore.FileInputData{Name: cmd.changeProcessor.Filename, Path: cmd.changeProcessor.Path, OnCloud: cmd.exisitingFileRef.OnCloud} + fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) + if err != nil { + return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) + } + + result.Filename = cmd.changeProcessor.Filename + result.Hash = fileOutputData.ContentHash + result.MerkleRoot = fileOutputData.MerkleRoot + result.Size = fileOutputData.Size + + if len(cmd.changeProcessor.Hash) > 0 && cmd.changeProcessor.Hash != fileOutputData.ContentHash { + return result, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") + } + if len(cmd.changeProcessor.MerkleRoot) > 0 && cmd.changeProcessor.MerkleRoot != fileOutputData.MerkleRoot { + return result, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") + } + if fileOutputData.Size > config.Configuration.MaxFileSize { + return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") + } + + cmd.changeProcessor.Hash = fileOutputData.ContentHash + cmd.changeProcessor.MerkleRoot = fileOutputData.MerkleRoot + cmd.changeProcessor.AllocationID = allocationObj.ID + cmd.changeProcessor.Size = fileOutputData.Size + + allocationSize := fileOutputData.Size + + if allocationObj.BlobberSizeUsed+(allocationSize-cmd.exisitingFileRef.Size) > allocationObj.BlobberSize { + return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") + } + + cmd.allocationChange = &allocation.AllocationChange{} + cmd.allocationChange.ConnectionID = connectionObj.ConnectionID + cmd.allocationChange.Size = allocationSize - cmd.exisitingFileRef.Size + cmd.allocationChange.Operation = constants.FileOperationUpdate + + connectionObj.Size += cmd.allocationChange.Size + + return result, nil + +} + +// ProcessThumbnail flush thumbnail file to FileStorage if it has. +func (cmd *UpdateFileCMD) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { + + thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") + + if thumbHeader != nil { + + defer thumbfile.Close() + + thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.changeProcessor.Path} + thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, thumbInputData, thumbfile, connectionObj.ConnectionID) + if err != nil { + return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) + } + if len(cmd.changeProcessor.ThumbnailHash) > 0 && cmd.changeProcessor.ThumbnailHash != thumbOutputData.ContentHash { + return common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") + } + cmd.changeProcessor.ThumbnailHash = thumbOutputData.ContentHash + cmd.changeProcessor.ThumbnailSize = thumbOutputData.Size + cmd.changeProcessor.ThumbnailFilename = thumbInputData.Name + } + + return nil + +} + +// UpdateChange add UpdateFileChanger in db +func (cmd *UpdateFileCMD) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { + connectionObj.AddChange(cmd.allocationChange, cmd.changeProcessor) + + return connectionObj.Save(ctx) +} diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 80f7e4439..460bcd98c 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -19,17 +19,17 @@ import ( "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/constants" "github.com/gorilla/mux" ) var storageHandler StorageHandler -func GetMetaDataStore() *datastore.Store { +func GetMetaDataStore() datastore.Store { return datastore.GetStore() } @@ -114,14 +114,14 @@ func WithConnection(handler common.JSONResponderF) common.JSONResponderF { func setupHandlerContext(ctx context.Context, r *http.Request) context.Context { var vars = mux.Vars(r) - ctx = context.WithValue(ctx, constants.CLIENT_CONTEXT_KEY, + ctx = context.WithValue(ctx, constants.ContextKeyClient, r.Header.Get(common.ClientHeader)) - ctx = context.WithValue(ctx, constants.CLIENT_KEY_CONTEXT_KEY, + ctx = context.WithValue(ctx, constants.ContextKeyClientKey, r.Header.Get(common.ClientKeyHeader)) - ctx = context.WithValue(ctx, constants.ALLOCATION_CONTEXT_KEY, + ctx = context.WithValue(ctx, constants.ContextKeyAllocation, vars["allocation"]) // signature is not requered for all requests, but if header is empty it won`t affect anything - ctx = context.WithValue(ctx, constants.CLIENT_SIGNATURE_HEADER_KEY, r.Header.Get(common.ClientSignatureHeader)) + ctx = context.WithValue(ctx, constants.ContextKeyClientSignatureHeaderKey, r.Header.Get(common.ClientSignatureHeader)) return ctx } @@ -354,7 +354,7 @@ func CleanupDiskHandler(ctx context.Context, r *http.Request) (interface{}, erro func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { ctx = setupHandlerContext(ctx, r) - allocationID := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationID := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := storageHandler.verifyAllocation(ctx, allocationID, true) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) @@ -377,7 +377,7 @@ func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if clientID != allocationObj.OwnerID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -405,7 +405,7 @@ func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { ctx = setupHandlerContext(ctx, r) - allocationID := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationID := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := storageHandler.verifyAllocation(ctx, allocationID, true) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) diff --git a/code/go/0chain.net/blobbercore/handler/handler_integration_tests.go b/code/go/0chain.net/blobbercore/handler/handler_integration_tests.go index 9f67e0ab5..f5f1f4d6a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_integration_tests.go +++ b/code/go/0chain.net/blobbercore/handler/handler_integration_tests.go @@ -14,11 +14,11 @@ import ( "runtime/pprof" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/gosdk/constants" "github.com/gorilla/mux" @@ -28,7 +28,7 @@ import ( var storageHandler StorageHandler -func GetMetaDataStore() *datastore.Store { +func GetMetaDataStore() datastore.Store { return datastore.GetStore() } @@ -96,11 +96,11 @@ func WithConnection(handler common.JSONResponderF) common.JSONResponderF { func setupHandlerContext(ctx context.Context, r *http.Request) context.Context { var vars = mux.Vars(r) - ctx = context.WithValue(ctx, constants.CLIENT_CONTEXT_KEY, + ctx = context.WithValue(ctx, constants.ContextKeyClient, r.Header.Get(common.ClientHeader)) - ctx = context.WithValue(ctx, constants.CLIENT_KEY_CONTEXT_KEY, + ctx = context.WithValue(ctx, constants.ContextKeyClientKey, r.Header.Get(common.ClientKeyHeader)) - ctx = context.WithValue(ctx, constants.ALLOCATION_CONTEXT_KEY, + ctx = context.WithValue(ctx, constants.ContextKeyAllocation, vars["allocation"]) return ctx } diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 25aa3f045..7d0a3ecd2 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -50,8 +50,7 @@ func (MockFileBlockGetter) GetFileBlock( allocationID string, fileData *filestore.FileInputData, blockNum int64, - numBlocks int64, -) ([]byte, error) { + numBlocks int64) ([]byte, error) { return []byte(mockFileBlock), nil } @@ -68,7 +67,7 @@ var encscheme zencryption.EncryptionScheme func setupEncryptionScheme() { encscheme = zencryption.NewEncryptionScheme() mnemonic := client.GetClient().Mnemonic - if err := encscheme.Initialize(mnemonic); err != nil { + if _, err := encscheme.Initialize(mnemonic); err != nil { panic("initialize encscheme") } encscheme.InitForEncryption("filetype:audio") @@ -1014,7 +1013,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { } q := url.Query() - formFieldByt, err := json.Marshal(&allocation.UpdateFileChange{}) + formFieldByt, err := json.Marshal(&allocation.UpdateFileChanger{}) if err != nil { t.Fatal(err) } diff --git a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go index 6a5f0e18c..3a744b396 100644 --- a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go @@ -4,13 +4,13 @@ import ( "context" "database/sql" "fmt" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "log" "math/rand" "os" "strings" "time" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "google.golang.org/grpc" "gorm.io/driver/postgres" diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index b2c1b26de..e1bf32d2d 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -20,7 +20,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" @@ -30,6 +29,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/gosdk/constants" zfileref "github.com/0chain/gosdk/zboxcore/fileref" "gorm.io/datatypes" @@ -185,9 +185,9 @@ func (fsh *StorageHandler) DownloadFile( // get client and allocation ids var ( - clientID = ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) - allocationTx = ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) // runtime type check + clientID = ctx.Value(constants.ContextKeyClient).(string) + allocationTx = ctx.Value(constants.ContextKeyAllocation).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) // runtime type check alloc *allocation.Allocation ) @@ -396,6 +396,7 @@ func (fsh *StorageHandler) DownloadFile( fileData.Path = fileref.Path fileData.Hash = fileref.ThumbnailHash fileData.OnCloud = fileref.OnCloud + fileData.ChunkSize = fileref.ChunkSize respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, blockNum, numBlocks) if err != nil { @@ -408,6 +409,8 @@ func (fsh *StorageHandler) DownloadFile( fileData.Path = fileref.Path fileData.Hash = fileref.ContentHash fileData.OnCloud = fileref.OnCloud + fileData.ChunkSize = fileref.ChunkSize + respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, blockNum, numBlocks) if err != nil { @@ -451,7 +454,7 @@ func (fsh *StorageHandler) DownloadFile( // reEncrypt does not require pub / private key, // we could probably make it a classless function - if err := encscheme.Initialize(""); err != nil { + if _, err := encscheme.Initialize(""); err != nil { return nil, err } if err := encscheme.InitForDecryption("filetype:audio", fileref.EncryptedKey); err != nil { @@ -502,9 +505,9 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use POST instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) - clientKey := ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) + clientKey := ctx.Value(constants.ContextKeyClientKey).(string) clientKeyBytes, _ := hex.DecodeString(clientKey) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) @@ -539,8 +542,8 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b var isCollaborator bool for _, change := range connectionObj.Changes { - if change.Operation == allocation.UPDATE_OPERATION { - updateFileChange := new(allocation.UpdateFileChange) + if change.Operation == constants.FileOperationUpdate { + updateFileChange := new(allocation.UpdateFileChanger) if err := updateFileChange.Unmarshal(change.Input); err != nil { return nil, err } @@ -624,6 +627,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b if err != nil { return nil, err } + allocationRoot := encryption.Hash(rootRef.Hash + ":" + strconv.FormatInt(int64(writeMarker.Timestamp), 10)) if allocationRoot != writeMarker.AllocationRoot { @@ -677,7 +681,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) @@ -689,8 +693,8 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i allocationID := allocationObj.ID - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), allocationObj.OwnerPublicKey) if !valid || err != nil { @@ -738,7 +742,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i allocationChange := &allocation.AllocationChange{} allocationChange.ConnectionID = connectionObj.ConnectionID allocationChange.Size = 0 - allocationChange.Operation = allocation.RENAME_OPERATION + allocationChange.Operation = constants.FileOperationRename dfc := &allocation.RenameFileChange{ConnectionID: connectionObj.ConnectionID, AllocationID: connectionObj.AllocationID, Path: objectRef.Path} dfc.NewName = new_name @@ -769,8 +773,8 @@ func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, } var ( - allocTx = ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) - clientID = ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + allocTx = ctx.Value(constants.ContextKeyAllocation).(string) + clientID = ctx.Value(constants.ContextKeyClient).(string) alloc *allocation.Allocation ) @@ -790,7 +794,7 @@ func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, } // runtime type check - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) if clientID == "" { return nil, common.NewError("update_object_attributes", @@ -847,7 +851,7 @@ func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, var change = new(allocation.AllocationChange) change.ConnectionID = conn.ConnectionID - change.Operation = allocation.UPDATE_ATTRS_OPERATION + change.Operation = constants.FileOperationUpdateAttrs var uafc = &allocation.AttributesChange{ ConnectionID: conn.ConnectionID, @@ -874,7 +878,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) @@ -889,8 +893,8 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int return nil, common.NewError("immutable_allocation", "Cannot copy data in an immutable allocation") } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) allocationID := allocationObj.ID @@ -945,7 +949,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int allocationChange := &allocation.AllocationChange{} allocationChange.ConnectionID = connectionObj.ConnectionID allocationChange.Size = objectRef.Size - allocationChange.Operation = allocation.COPY_OPERATION + allocationChange.Operation = constants.FileOperationCopy dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ConnectionID, AllocationID: connectionObj.AllocationID, DestPath: destPath} dfc.SrcPath = objectRef.Path @@ -974,14 +978,14 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn } fileRef, _ := reference.GetReference(ctx, connectionObj.AllocationID, path) - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) if fileRef != nil { deleteSize := fileRef.Size allocationChange := &allocation.AllocationChange{} allocationChange.ConnectionID = connectionObj.ConnectionID allocationChange.Size = 0 - deleteSize - allocationChange.Operation = allocation.DELETE_OPERATION + allocationChange.Operation = constants.FileOperationDelete dfc := &allocation.DeleteFileChange{ConnectionID: connectionObj.ConnectionID, AllocationID: connectionObj.AllocationID, Name: fileRef.Name, Hash: fileRef.Hash, Path: fileRef.Path, Size: deleteSize} @@ -1002,8 +1006,8 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn } func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blobberhttp.UploadResult, error) { - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { @@ -1052,7 +1056,7 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo allocationChange := &allocation.AllocationChange{} allocationChange.ConnectionID = connectionObj.ConnectionID allocationChange.Size = 0 - allocationChange.Operation = allocation.CREATEDIR_OPERATION + allocationChange.Operation = constants.FileOperationCreateDir connectionObj.Size += allocationChange.Size var formData allocation.NewFileChange formData.Filename = dirPath @@ -1087,11 +1091,11 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blobberhttp.UploadResult, error) { if r.Method == "GET" { - return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use multi-part form POST / PUT / DELETE instead") + return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use multi-part form POST / PUT / DELETE / PATCH instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { @@ -1105,7 +1109,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo publicKey := allocationObj.OwnerPublicKey if isCollaborator { - publicKey = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + publicKey = ctx.Value(constants.ContextKeyClientKey).(string) } valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), publicKey) @@ -1132,6 +1136,14 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } + cmd := createFileCommand(r) + + err = cmd.IsAuthorized(ctx, r, allocationObj, clientID) + + if err != nil { + return nil, err + } + connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) if err != nil { return nil, common.NewError("meta_error", "Error reading metadata for connection") @@ -1141,151 +1153,52 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo mutex.Lock() defer mutex.Unlock() - result := &blobberhttp.UploadResult{} - - if fileOperation == allocation.DELETE_OPERATION { - if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") - } - result, err = fsh.DeleteFile(ctx, r, connectionObj) - if err != nil { - return nil, err - } - } else if fileOperation == allocation.INSERT_OPERATION || fileOperation == allocation.UPDATE_OPERATION { - formField := getFormFieldName(fileOperation) - var formData allocation.UpdateFileChange - uploadMetaString := r.FormValue(formField) - err = json.Unmarshal([]byte(uploadMetaString), &formData) - if err != nil { - return nil, common.NewError("invalid_parameters", - "Invalid parameters. Error parsing the meta data for upload."+err.Error()) - } - existingFileRefSize := int64(0) - existingFileOnCloud := false - if fileOperation == allocation.INSERT_OPERATION { - if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") - } - - if existingFileRef != nil { - return nil, common.NewError("duplicate_file", "File at path already exists") - } - } else if fileOperation == allocation.UPDATE_OPERATION { - if existingFileRef == nil { - return nil, common.NewError("invalid_file_update", "File at path does not exist for update") - } - - if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID && !isCollaborator { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner, collaborator or the payer of the allocation") - } - } - - if existingFileRef != nil { - existingFileRefSize = existingFileRef.Size - existingFileOnCloud = existingFileRef.OnCloud - } - - origfile, _, err := r.FormFile("uploadFile") - if err != nil { - return nil, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) - } - defer origfile.Close() - - thumbfile, thumbHeader, _ := r.FormFile("uploadThumbnailFile") - thumbnailPresent := thumbHeader != nil - if thumbnailPresent { - defer thumbfile.Close() - } + result, err := cmd.ProcessContent(ctx, r, allocationObj, connectionObj) - fileInputData := &filestore.FileInputData{Name: formData.Filename, Path: formData.Path, OnCloud: existingFileOnCloud} - fileOutputData, err := filestore.GetFileStore().WriteFile(allocationID, fileInputData, origfile, connectionObj.ConnectionID) - if err != nil { - return nil, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) - } - - result.Filename = formData.Filename - result.Hash = fileOutputData.ContentHash - result.MerkleRoot = fileOutputData.MerkleRoot - result.Size = fileOutputData.Size - - if len(formData.Hash) > 0 && formData.Hash != fileOutputData.ContentHash { - return nil, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") - } - if len(formData.MerkleRoot) > 0 && formData.MerkleRoot != fileOutputData.MerkleRoot { - return nil, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") - } - if fileOutputData.Size > config.Configuration.MaxFileSize { - return nil, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") - } - - formData.Hash = fileOutputData.ContentHash - formData.MerkleRoot = fileOutputData.MerkleRoot - formData.AllocationID = allocationID - formData.Size = fileOutputData.Size + if err != nil { + return nil, err + } - allocationSize := fileOutputData.Size - if thumbnailPresent { - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: formData.Path} - thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationID, thumbInputData, thumbfile, connectionObj.ConnectionID) - if err != nil { - return nil, common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) - } - if len(formData.ThumbnailHash) > 0 && formData.ThumbnailHash != thumbOutputData.ContentHash { - return nil, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") - } - formData.ThumbnailHash = thumbOutputData.ContentHash - formData.ThumbnailSize = thumbOutputData.Size - formData.ThumbnailFilename = thumbInputData.Name - } + err = cmd.ProcessThumbnail(ctx, r, allocationObj, connectionObj) - if allocationObj.BlobberSizeUsed+(allocationSize-existingFileRefSize) > allocationObj.BlobberSize { - return nil, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") - } + if err != nil { + return nil, err + } - allocationChange := &allocation.AllocationChange{} - allocationChange.ConnectionID = connectionObj.ConnectionID - allocationChange.Size = allocationSize - existingFileRefSize - allocationChange.Operation = fileOperation + err = cmd.UpdateChange(ctx, connectionObj) - connectionObj.Size += allocationChange.Size - if fileOperation == allocation.INSERT_OPERATION { - connectionObj.AddChange(allocationChange, &formData.NewFileChange) - } else if fileOperation == allocation.UPDATE_OPERATION { - connectionObj.AddChange(allocationChange, &formData) - } - } - err = connectionObj.Save(ctx) if err != nil { Logger.Error("Error in writing the connection meta data", zap.Error(err)) - return nil, common.NewError("connection_write_error", "Error writing the connection meta data") + return nil, common.NewError("connection_write_error", err.Error()) //"Error writing the connection meta data") } - return result, nil + return &result, nil } func getFormFieldName(mode string) string { - formField := "uploadMeta" - if mode == allocation.UPDATE_OPERATION { - formField = "updateMeta" - } + return "uploadMeta" + // formField := "uploadMeta" + // if mode == constants.FileOperationUpdate { + // //formField = "updateMeta" + // } - return formField + //return formField } func getFileOperation(r *http.Request) string { - mode := allocation.INSERT_OPERATION + mode := constants.FileOperationInsert if r.Method == "PUT" { - mode = allocation.UPDATE_OPERATION + mode = constants.FileOperationUpdate } else if r.Method == "DELETE" { - mode = allocation.DELETE_OPERATION + mode = constants.FileOperationDelete } return mode } func getExistingFileRef(fsh *StorageHandler, ctx context.Context, r *http.Request, allocationObj *allocation.Allocation, fileOperation string) *reference.Ref { - if fileOperation == allocation.INSERT_OPERATION || fileOperation == allocation.UPDATE_OPERATION { - var formData allocation.UpdateFileChange + if fileOperation == constants.FileOperationInsert || fileOperation == constants.FileOperationUpdate { + var formData allocation.UpdateFileChanger uploadMetaString := r.FormValue(getFormFieldName(fileOperation)) err := json.Unmarshal([]byte(uploadMetaString), &formData) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go new file mode 100644 index 000000000..2020ebaa9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go @@ -0,0 +1,198 @@ +package handler + +import ( + "context" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/mock" + "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk/zboxcore/sdk" +) + +func BenchmarkUploadFileWithDisk(b *testing.B) { + + KB := 1024 + MB := 1024 * KB + //GB := 1024 * MB + + datastore.UseMocket(false) + blobber := mock.NewBlobberClient() + + allocationID := "benchmark_uploadfile" + + allocation := map[string]interface{}{ + "id": allocationID, + "tx": allocationID, + "size": 1024 * 1024 * 100, + "blobber_size": 1024 * 1024 * 1000, + "owner_id": blobber.ClientID, + "owner_public_key": blobber.Wallet.Keys[0].PublicKey, + "expiration_date": time.Now().Add(24 * time.Hour).Unix(), + } + + mock.MockGetAllocationByID(allocationID, allocation) + + formBuilder := sdk.CreateChunkedUploadFormBuilder() + + var storageHandler StorageHandler + + benchmarks := []struct { + Name string + // Size int + ChunkSize int + }{ + {Name: "64K", ChunkSize: 64 * KB}, + {Name: "640K", ChunkSize: 640 * KB}, + {Name: "6M", ChunkSize: 6 * MB}, + {Name: "60M", ChunkSize: 60 * MB}, + } + + for _, bm := range benchmarks { + b.Run(bm.Name, func(b *testing.B) { + + fileName := strings.Replace(bm.Name, " ", "_", -1) + ".txt" + chunkBytes := mock.GenerateRandomBytes(bm.ChunkSize) + fileMeta := &sdk.FileMeta{ + Path: "/tmp/" + fileName, + ActualSize: int64(bm.ChunkSize), + + MimeType: "plain/text", + RemoteName: fileName, + RemotePath: "/" + fileName, + Attributes: fileref.Attributes{}, + } + + hasher := sdk.CreateHasher(bm.ChunkSize) + isFinal := false + + body, formData, _ := formBuilder.Build(fileMeta, hasher, strconv.FormatInt(time.Now().UnixNano(), 10), int64(bm.ChunkSize), 0, isFinal, "", chunkBytes, nil) + + req, err := blobber.NewRequest(http.MethodPost, "http://127.0.0.1:5051/v1/file/upload/benchmark_upload", body) + + if err != nil { + b.Fatal(err) + return + } + + req.Header.Set("Content-Type", formData.ContentType) + err = blobber.SignRequest(req, allocationID) + if err != nil { + b.Fatal(err) + return + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + ctx := GetMetaDataStore().CreateTransaction(context.TODO()) + ctx = mock.SetupHandlerContext(ctx, req, allocationID) + _, err := storageHandler.WriteFile(ctx, req) + + if err != nil { + b.Fatal(err) + return + } + + } + }) + } + +} + +func BenchmarkUploadFileWithNoDisk(b *testing.B) { + + KB := 1024 + MB := 1024 * KB + //GB := 1024 * MB + + datastore.UseMocket(false) + filestore.UseMock() + blobber := mock.NewBlobberClient() + + allocationID := "benchmark_uploadfile" + + allocation := map[string]interface{}{ + "id": allocationID, + "tx": allocationID, + "size": 1024 * 1024 * 100, + "blobber_size": 1024 * 1024 * 1000, + "owner_id": blobber.ClientID, + "owner_public_key": blobber.Wallet.Keys[0].PublicKey, + "expiration_date": time.Now().Add(24 * time.Hour).Unix(), + } + + mock.MockGetAllocationByID(allocationID, allocation) + + formBuilder := sdk.CreateChunkedUploadFormBuilder() + + var storageHandler StorageHandler + + benchmarks := []struct { + Name string + // Size int + ChunkSize int + }{ + {Name: "64K", ChunkSize: 64 * KB}, + {Name: "640K", ChunkSize: 640 * KB}, + {Name: "6M", ChunkSize: 6 * MB}, + {Name: "60M", ChunkSize: 60 * MB}, + } + + for _, bm := range benchmarks { + b.Run(bm.Name, func(b *testing.B) { + + fileName := strings.Replace(bm.Name, " ", "_", -1) + ".txt" + chunkBytes := mock.GenerateRandomBytes(bm.ChunkSize) + fileMeta := &sdk.FileMeta{ + Path: "/tmp/" + fileName, + ActualSize: int64(bm.ChunkSize), + + MimeType: "plain/text", + RemoteName: fileName, + RemotePath: "/" + fileName, + Attributes: fileref.Attributes{}, + } + + hasher := sdk.CreateHasher(bm.ChunkSize) + isFinal := false + + body, formData, _ := formBuilder.Build(fileMeta, hasher, strconv.FormatInt(time.Now().UnixNano(), 10), int64(bm.ChunkSize), 0, isFinal, "", chunkBytes, nil) + + req, err := blobber.NewRequest(http.MethodPost, "http://127.0.0.1:5051/v1/file/upload/benchmark_upload", body) + + if err != nil { + b.Fatal(err) + return + } + + req.Header.Set("Content-Type", formData.ContentType) + err = blobber.SignRequest(req, allocationID) + if err != nil { + b.Fatal(err) + return + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + + ctx := GetMetaDataStore().CreateTransaction(context.TODO()) + + ctx = mock.SetupHandlerContext(ctx, req, allocationID) + _, err := storageHandler.WriteFile(ctx, req) + + if err != nil { + b.Fatal(err) + return + } + + } + }) + } + +} diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index 7a861f89c..66c910ded 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -31,8 +31,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/gosdk/constants" "github.com/stretchr/testify/require" "testing" @@ -40,19 +40,19 @@ import ( func TestDownloadFile(t *testing.T) { const ( - mocketLogging = false - mockBlobberId = "mock_blobber_id" - mockAllocationId = "mock_allocation_id" - mockAllocationTx = "mock_allocation_Tx" + mocketLogging = false + mockBlobberId = "mock_blobber_id" + mockAllocationId = "mock_allocation_id" + mockAllocationTx = "mock_allocation_Tx" mockRemoteFilePath = "mock/remote/file/path" mockBlockNumber = 1 mockEncryptKey = "mock encrypt key" - mockClientWallet = "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}" - mockOwnerWallet = "{\"client_id\":\"5d0229e0141071c1f88785b1faba4b612582f9d446b02e8d893f1e0d0ce92cdc\",\"client_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"keys\":[{\"public_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"private_key\":\"4f8af6fb1098a3817d705aef96db933f31755674b00a5d38bb2439c0a27b0117\"}],\"mnemonics\":\"erode transfer noble civil ridge cloth sentence gauge board wheel sight caution okay sand ranch ice frozen frown grape lion feed fox game zone\",\"version\":\"1.0\",\"date_created\":\"2021-09-04T14:11:06+01:00\"}" - mockReadPrice = int64(0.1 * 1e10) - mockWritePrice = int64(0.5 * 1e10) - mockBigBalance = int64(10000 * 1e10) - mockPoolId = "mock pool id" + mockClientWallet = "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}" + mockOwnerWallet = "{\"client_id\":\"5d0229e0141071c1f88785b1faba4b612582f9d446b02e8d893f1e0d0ce92cdc\",\"client_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"keys\":[{\"public_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"private_key\":\"4f8af6fb1098a3817d705aef96db933f31755674b00a5d38bb2439c0a27b0117\"}],\"mnemonics\":\"erode transfer noble civil ridge cloth sentence gauge board wheel sight caution okay sand ranch ice frozen frown grape lion feed fox game zone\",\"version\":\"1.0\",\"date_created\":\"2021-09-04T14:11:06+01:00\"}" + mockReadPrice = int64(0.1 * 1e10) + mockWritePrice = int64(0.5 * 1e10) + mockBigBalance = int64(10000 * 1e10) + mockPoolId = "mock pool id" ) ts := time.Now().Add(time.Hour) var mockLongTimeInFuture = common.Timestamp(ts.Unix()) + common.Timestamp(time.Second*1000) @@ -405,12 +405,12 @@ func TestDownloadFile(t *testing.T) { setupCtx := func(p parameters) context.Context { ctx := context.TODO() - ctx = context.WithValue(ctx, constants.CLIENT_CONTEXT_KEY, client.GetClientID()) - ctx = context.WithValue(ctx, constants.ALLOCATION_CONTEXT_KEY, p.inData.allocationTx) - ctx = context.WithValue(ctx, constants.CLIENT_KEY_CONTEXT_KEY, client.GetClientPublicKey()) + ctx = context.WithValue(ctx, constants.ContextKeyClient, client.GetClientID()) + ctx = context.WithValue(ctx, constants.ContextKeyAllocation, p.inData.allocationTx) + ctx = context.WithValue(ctx, constants.ContextKeyClientKey, client.GetClientPublicKey()) db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.CONNECTION_CONTEXT_KEY, db) + ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) return ctx } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 1b682fd3f..bed0323ea 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -10,7 +10,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" @@ -18,19 +17,21 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/constants" "go.uber.org/zap" ) const ( FormFileParseMaxMemory = 10 * 1024 * 1024 OffsetDateLayout = "2006-01-02T15:04:05.99999Z07:00" - DownloadCcontentFull = "full" + DownloadContentFull = "full" DownloadContentThumb = "thumbnail" - PageLimit = 100 //100 rows will make upto 100 KB + PageLimit = 100 //100 rows will make up to 100 KB ) type StorageHandler struct{} +// verifyAllocation try to get allocation from postgres.if it doesn't exists, get it from sharders, and insert it into postgres. func (fsh *StorageHandler) verifyAllocation(ctx context.Context, tx string, readonly bool) (alloc *allocation.Allocation, err error) { @@ -121,7 +122,7 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) alloc, err := fsh.verifyAllocation(ctx, allocationTx, true) if err != nil { @@ -129,12 +130,12 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in } allocationID := alloc.ID - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { @@ -191,7 +192,7 @@ func (fsh *StorageHandler) AddCommitMetaTxn(ctx context.Context, r *http.Request if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, true) if err != nil { @@ -199,12 +200,12 @@ func (fsh *StorageHandler) AddCommitMetaTxn(ctx context.Context, r *http.Request } allocationID := allocationObj.ID - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { @@ -252,21 +253,21 @@ func (fsh *StorageHandler) AddCommitMetaTxn(ctx context.Context, r *http.Request } func (fsh *StorageHandler) AddCollaborator(ctx context.Context, r *http.Request) (interface{}, error) { - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, true) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } - clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string) + clientSign, _ := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } allocationID := allocationObj.ID - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { @@ -338,25 +339,25 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, true) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } allocationID := allocationObj.ID - clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string) + clientSign, _ := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - _ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { @@ -395,8 +396,8 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* if r.Method == "POST" { return nil, common.NewError("invalid_method", "Invalid method used. Use GET instead") } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, true) if err != nil { @@ -475,7 +476,7 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request return } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { errCh <- common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) @@ -483,14 +484,14 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request } allocationID := allocationObj.ID - clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string) + clientSign, _ := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) if !valid || err != nil { errCh <- common.NewError("invalid_signature", "Invalid signature") return } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 { errCh <- common.NewError("invalid_operation", "Please pass clientID in the header") return @@ -509,8 +510,10 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request } refPath := &reference.ReferencePath{Ref: rootRef} - refsToProcess := make([]*reference.ReferencePath, 0) - refsToProcess = append(refsToProcess, refPath) + + refsToProcess := []*reference.ReferencePath{refPath} + + //convert Ref tree to ReferencePath tree for len(refsToProcess) > 0 { refToProcess := refsToProcess[0] refToProcess.Meta = refToProcess.Ref.GetListingData(ctx) @@ -548,20 +551,20 @@ func (fsh *StorageHandler) GetObjectPath(ctx context.Context, r *http.Request) ( if r.Method == "POST" { return nil, common.NewError("invalid_method", "Invalid method used. Use GET instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } allocationID := allocationObj.ID - clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string) + clientSign, _ := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -606,7 +609,7 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( if r.Method == "POST" { return nil, common.NewError("invalid_method", "Invalid method used. Use GET instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { @@ -614,13 +617,13 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( } allocationID := allocationObj.ID - clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string) + clientSign, _ := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -673,21 +676,21 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( //Updated gives rows that is updated compared to the date given. And deleted gives deleted refs compared to the date given. //Updated date time format should be as declared in above constant; OffsetDateLayout func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobberhttp.RefResult, error) { - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } - clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string) + clientSign, _ := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } allocationID := allocationObj.ID - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -786,7 +789,7 @@ func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) ( if r.Method != "POST" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") } - allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { @@ -794,7 +797,7 @@ func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) ( } allocationID := allocationObj.ID - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/storage_handler_bench_test.go new file mode 100644 index 000000000..abeebd162 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/storage_handler_bench_test.go @@ -0,0 +1 @@ +package handler diff --git a/code/go/0chain.net/blobbercore/handler/upload_integration_test.go b/code/go/0chain.net/blobbercore/handler/upload_integration_test.go index 08f7f4a09..b06d7c048 100644 --- a/code/go/0chain.net/blobbercore/handler/upload_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/upload_integration_test.go @@ -4,11 +4,12 @@ import ( "context" "encoding/hex" "encoding/json" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "io" "os" "testing" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" @@ -24,7 +25,7 @@ func TestBlobberGRPCService_UploadFile(t *testing.T) { pubKeyBytes, _ := hex.DecodeString(pubKey) clientId := encryption.Hash(pubKeyBytes) - formFieldByt, err := json.Marshal(&allocation.UpdateFileChange{NewFileChange: allocation.NewFileChange{Filename: `helper_integration_test.go`}}) + formFieldByt, err := json.Marshal(&allocation.UpdateFileChanger{BaseFileChanger: allocation.BaseFileChanger{Filename: `helper_integration_test.go`}}) if err != nil { t.Fatal(err) } diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 4a086c402..3aa705f1d 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -20,9 +20,9 @@ import ( ) func SetupWorkers(ctx context.Context) { - go CleanupTempFiles(ctx) + go startCleanupTempFiles(ctx) if config.Configuration.MinioStart { - go MoveColdDataToCloud(ctx) + go startMoveColdDataToCloud(ctx) } } @@ -52,7 +52,40 @@ func CleanupDiskFiles(ctx context.Context) error { return nil } -func CleanupTempFiles(ctx context.Context) { +func cleanupTempFiles(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + Logger.Error("[recover] cleanupTempFiles", zap.Any("err", r)) + } + }() + + rctx := datastore.GetStore().CreateTransaction(ctx) + db := datastore.GetStore().GetTransaction(rctx) + now := time.Now().UTC() + then := now.Add(time.Duration(-config.Configuration.OpenConnectionWorkerTolerance) * time.Second) + + var openConnectionsToDelete []allocation.AllocationChangeCollector + db.Table((&allocation.AllocationChangeCollector{}).TableName()).Where("updated_at < ? AND status IN (?,?)", then, allocation.NewConnection, allocation.InProgressConnection).Preload("Changes").Find(&openConnectionsToDelete) + for _, connection := range openConnectionsToDelete { + + Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ConnectionID)) + connection.ComputeProperties() + nctx := datastore.GetStore().CreateTransaction(ctx) + ndb := datastore.GetStore().GetTransaction(nctx) + for _, changeProcessor := range connection.AllocationChanges { + if err := changeProcessor.DeleteTempFile(); err != nil { + Logger.Error("AllocationChangeProcessor_DeleteTempFile", zap.Error(err)) + } + } + ndb.Model(connection).Updates(allocation.AllocationChangeCollector{Status: allocation.DeletedConnection}) + ndb.Commit() + nctx.Done() + } + db.Rollback() + rctx.Done() +} + +func startCleanupTempFiles(ctx context.Context) { var iterInprogress = false ticker := time.NewTicker(time.Duration(config.Configuration.OpenConnectionWorkerFreq) * time.Second) for { @@ -63,35 +96,71 @@ func CleanupTempFiles(ctx context.Context) { //Logger.Info("Trying to redeem writemarkers.", zap.Any("iterInprogress", iterInprogress), zap.Any("numOfWorkers", numOfWorkers)) if !iterInprogress { iterInprogress = true //nolint:ineffassign // probably has something to do with goroutines - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) - now := time.Now() - then := now.Add(time.Duration(-config.Configuration.OpenConnectionWorkerTolerance) * time.Second) - var openConnectionsToDelete []allocation.AllocationChangeCollector - db.Table((&allocation.AllocationChangeCollector{}).TableName()).Where("updated_at < ? AND status IN (?,?)", then, allocation.NewConnection, allocation.InProgressConnection).Preload("Changes").Find(&openConnectionsToDelete) - for _, connection := range openConnectionsToDelete { - Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ConnectionID)) - connection.ComputeProperties() - nctx := datastore.GetStore().CreateTransaction(ctx) - ndb := datastore.GetStore().GetTransaction(nctx) - for _, changeProcessor := range connection.AllocationChanges { - if err := changeProcessor.DeleteTempFile(); err != nil { - Logger.Error("AllocationChangeProcessor_DeleteTempFile", zap.Error(err)) - } - } - ndb.Model(connection).Updates(allocation.AllocationChangeCollector{Status: allocation.DeletedConnection}) - ndb.Commit() - nctx.Done() - } - db.Rollback() - rctx.Done() + cleanupTempFiles(ctx) iterInprogress = false } } } } -func MoveColdDataToCloud(ctx context.Context) { +func moveColdDataToCloud(ctx context.Context, coldStorageMinFileSize int64, limit int64) { + defer func() { + if r := recover(); r != nil { + Logger.Error("[recover] moveColdDataToCloud", zap.Any("err", r)) + } + }() + + fs := filestore.GetFileStore() + totalDiskSizeUsed, err := fs.GetTotalDiskSizeUsed() + if err != nil { + Logger.Error("Unable to get total disk size used from the file store", zap.Error(err)) + return + } + + // Check if capacity exceded the start capacity size + if totalDiskSizeUsed > config.Configuration.ColdStorageStartCapacitySize { + rctx := datastore.GetStore().CreateTransaction(ctx) + db := datastore.GetStore().GetTransaction(rctx) + // Get total number of fileRefs with size greater than limit and on_cloud = false + var totalRecords int64 + db.Table((&reference.Ref{}).TableName()). + Where("size > ? AND on_cloud = ?", coldStorageMinFileSize, false). + Count(&totalRecords) + + offset := int64(0) + for offset < totalRecords { + // Get all fileRefs with size greater than limit and on_cloud false + var fileRefs []*reference.Ref + db.Offset(int(offset)).Limit(int(limit)). + Table((&reference.Ref{}).TableName()). + Where("size > ? AND on_cloud = ?", coldStorageMinFileSize, false). + Find(&fileRefs) + + for _, fileRef := range fileRefs { + if fileRef.Type == reference.DIRECTORY { + continue + } + + fileStat, err := stats.GetFileStats(rctx, fileRef.ID) + if err != nil { + Logger.Error("Unable to find filestats for fileRef with", zap.Any("reID", fileRef.ID)) + continue + } + + timeToAdd := time.Duration(config.Configuration.ColdStorageTimeLimitInHours) * time.Hour + if fileStat.UpdatedAt.Before(time.Now().Add(-1 * timeToAdd)) { + Logger.Info("Moving file to cloud", zap.Any("path", fileRef.Path), zap.Any("allocation", fileRef.AllocationID)) + moveFileToCloud(ctx, fileRef) + } + } + offset = offset + limit + } + db.Commit() + rctx.Done() + } +} + +func startMoveColdDataToCloud(ctx context.Context) { var iterInprogress = false var coldStorageMinFileSize = config.Configuration.ColdStorageMinimumFileSize var limit = config.Configuration.ColdStorageJobQueryLimit @@ -102,54 +171,8 @@ func MoveColdDataToCloud(ctx context.Context) { return case <-ticker.C: if !iterInprogress { - fs := filestore.GetFileStore() - totalDiskSizeUsed, err := fs.GetTotalDiskSizeUsed() - if err != nil { - Logger.Error("Unable to get total disk size used from the file store", zap.Error(err)) - return - } - // Check if capacity exceded the start capacity size - if totalDiskSizeUsed > config.Configuration.ColdStorageStartCapacitySize { - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) - // Get total number of fileRefs with size greater than limit and on_cloud = false - var totalRecords int64 - db.Table((&reference.Ref{}).TableName()). - Where("size > ? AND on_cloud = ?", coldStorageMinFileSize, false). - Count(&totalRecords) - - offset := int64(0) - for offset < totalRecords { - // Get all fileRefs with size greater than limit and on_cloud false - var fileRefs []*reference.Ref - db.Offset(int(offset)).Limit(int(limit)). - Table((&reference.Ref{}).TableName()). - Where("size > ? AND on_cloud = ?", coldStorageMinFileSize, false). - Find(&fileRefs) - - for _, fileRef := range fileRefs { - if fileRef.Type == reference.DIRECTORY { - continue - } - - fileStat, err := stats.GetFileStats(rctx, fileRef.ID) - if err != nil { - Logger.Error("Unable to find filestats for fileRef with", zap.Any("reID", fileRef.ID)) - continue - } - - timeToAdd := time.Duration(config.Configuration.ColdStorageTimeLimitInHours) * time.Hour - if fileStat.UpdatedAt.Before(time.Now().Add(-1 * timeToAdd)) { - Logger.Info("Moving file to cloud", zap.Any("path", fileRef.Path), zap.Any("allocation", fileRef.AllocationID)) - moveFileToCloud(ctx, fileRef) - } - } - offset = offset + limit - } - db.Commit() - rctx.Done() - } + moveColdDataToCloud(ctx, coldStorageMinFileSize, limit) iterInprogress = false stats.LastMinioScan = time.Now() Logger.Info("Move cold data to cloud worker running successfully") diff --git a/code/go/0chain.net/blobbercore/mock/allocation.go b/code/go/0chain.net/blobbercore/mock/allocation.go new file mode 100644 index 000000000..5d1896cf4 --- /dev/null +++ b/code/go/0chain.net/blobbercore/mock/allocation.go @@ -0,0 +1,12 @@ +package mock + +import ( + "github.com/selvatico/go-mocket" +) + +func MockGetAllocationByID(allocationID string, allocation map[string]interface{}) { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "allocations" WHERE "allocations"."tx" = $1 ORDER BY "allocations"."id" LIMIT 1`). + WithArgs(allocationID). + WithReply([]map[string]interface{}{allocation}) +} diff --git a/code/go/0chain.net/blobbercore/mock/buf.go b/code/go/0chain.net/blobbercore/mock/buf.go new file mode 100644 index 000000000..b9f161855 --- /dev/null +++ b/code/go/0chain.net/blobbercore/mock/buf.go @@ -0,0 +1,14 @@ +package mock + +import "crypto/rand" + +func GenerateRandomBytes(n int) []byte { + b := make([]byte, n) + _, err := rand.Read(b) + // Note that err == nil only if we read len(b) bytes. + if err != nil { + return nil + } + + return b +} diff --git a/code/go/0chain.net/blobbercore/mock/ctx.go b/code/go/0chain.net/blobbercore/mock/ctx.go new file mode 100644 index 000000000..7573aa48f --- /dev/null +++ b/code/go/0chain.net/blobbercore/mock/ctx.go @@ -0,0 +1,22 @@ +package mock + +import ( + "context" + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/gosdk/constants" +) + +func SetupHandlerContext(ctx context.Context, r *http.Request, allocation string) context.Context { + + ctx = context.WithValue(ctx, constants.ContextKeyClient, + r.Header.Get(common.ClientHeader)) + ctx = context.WithValue(ctx, constants.ContextKeyClientKey, + r.Header.Get(common.ClientKeyHeader)) + ctx = context.WithValue(ctx, constants.ContextKeyAllocation, + allocation) + // signature is not required for all requests, but if header is empty it won`t affect anything + ctx = context.WithValue(ctx, constants.ContextKeyClientSignatureHeaderKey, r.Header.Get(common.ClientSignatureHeader)) + return ctx +} diff --git a/code/go/0chain.net/blobbercore/mock/init.go b/code/go/0chain.net/blobbercore/mock/init.go new file mode 100644 index 000000000..768e21bd9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/mock/init.go @@ -0,0 +1,35 @@ +package mock + +import ( + "io" + "net/http" + "net/http/httptest" + + "github.com/0chain/gosdk/sdks" + "github.com/0chain/gosdk/sdks/blobber" +) + +const ( + zboxWallet = "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}" +) + +// const ( +// mockOwnerWallet = "{\"client_id\":\"5d0229e0141071c1f88785b1faba4b612582f9d446b02e8d893f1e0d0ce92cdc\",\"client_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"keys\":[{\"public_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"private_key\":\"4f8af6fb1098a3817d705aef96db933f31755674b00a5d38bb2439c0a27b0117\"}],\"mnemonics\":\"erode transfer noble civil ridge cloth sentence gauge board wheel sight caution okay sand ranch ice frozen frown grape lion feed fox game zone\",\"version\":\"1.0\",\"date_created\":\"2021-09-04T14:11:06+01:00\"}" +// ) + +func NewBlobberClient() *blobber.Blobber { + + z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain") + err := z.InitWallet(zboxWallet) + if err != nil { + panic("mock: z.InitWallet " + err.Error()) + } + z.NewRequest = func(method, url string, body io.Reader) (*http.Request, error) { + return httptest.NewRequest(method, url, body), nil + } + return blobber.New(z, "http://127.0.0.1:5051/") +} + +func InitServer() { + //client.PopulateClient(mockOwnerWallet, "bls0chain") +} diff --git a/code/go/0chain.net/blobbercore/readmarker/protocol.go b/code/go/0chain.net/blobbercore/readmarker/protocol.go index fb2e08f4f..87795e6c1 100644 --- a/code/go/0chain.net/blobbercore/readmarker/protocol.go +++ b/code/go/0chain.net/blobbercore/readmarker/protocol.go @@ -6,7 +6,7 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" + "github.com/0chain/gosdk/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/chain" @@ -35,12 +35,12 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All return common.NewError("read_marker_validation_failed", "Read Marker is not for the blobber") } - clientPublicKey := ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + clientPublicKey := ctx.Value(constants.ContextKeyClientKey).(string) if len(clientPublicKey) == 0 || clientPublicKey != rm.LatestRM.ClientPublicKey { return common.NewError("read_marker_validation_failed", "Could not get the public key of the client") } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 || clientID != rm.LatestRM.ClientID { return common.NewError("read_marker_validation_failed", "Read Marker clientID does not match request clientID") } diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go new file mode 100644 index 000000000..665e442a9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go @@ -0,0 +1,56 @@ +package readmarker + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/zap" +) + +func redeemReadMarker(ctx context.Context) { + defer func() { + iterInprogress = false + if r := recover(); r != nil { + logging.Logger.Error("[recover] redeemReadMarker", zap.Any("err", r)) + } + + }() + + if !iterInprogress { + iterInprogress = true + rctx := datastore.GetStore().CreateTransaction(ctx) + db := datastore.GetStore().GetTransaction(rctx) + readMarkers := make([]*ReadMarkerEntity, 0) + rm := &ReadMarkerEntity{RedeemRequired: true} + db.Where(rm). // redeem_required = true + Where("counter <> suspend"). // and not suspended + Order("created_at ASC").Find(&readMarkers) + if len(readMarkers) > 0 { + swg := sizedwaitgroup.New(config.Configuration.RMRedeemNumWorkers) + for _, rmEntity := range readMarkers { + swg.Add() + go func(redeemCtx context.Context, rmEntity *ReadMarkerEntity) { + redeemCtx = datastore.GetStore().CreateTransaction(redeemCtx) + defer redeemCtx.Done() + err := RedeemReadMarker(redeemCtx, rmEntity) + if err != nil { + logging.Logger.Error("Error redeeming the read marker.", zap.Error(err)) + } + db := datastore.GetStore().GetTransaction(redeemCtx) + err = db.Commit().Error + if err != nil { + logging.Logger.Error("Error commiting the readmarker redeem", zap.Error(err)) + } + swg.Done() + }(ctx, rmEntity) + } + swg.Wait() + } + db.Rollback() + rctx.Done() + iterInprogress = false + } +} diff --git a/code/go/0chain.net/blobbercore/readmarker/worker.go b/code/go/0chain.net/blobbercore/readmarker/worker.go index c6a5fb3fc..7f30f838a 100644 --- a/code/go/0chain.net/blobbercore/readmarker/worker.go +++ b/code/go/0chain.net/blobbercore/readmarker/worker.go @@ -6,17 +6,15 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/chain" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - "github.com/remeh/sizedwaitgroup" "go.uber.org/zap" ) func SetupWorkers(ctx context.Context) { - go RedeemMarkers(ctx) + go startRedeemMarkers(ctx) } func RedeemReadMarker(ctx context.Context, rmEntity *ReadMarkerEntity) ( @@ -79,47 +77,14 @@ func RedeemReadMarker(ctx context.Context, rmEntity *ReadMarkerEntity) ( var iterInprogress = false -func RedeemMarkers(ctx context.Context) { +func startRedeemMarkers(ctx context.Context) { ticker := time.NewTicker(time.Duration(config.Configuration.RMRedeemFreq) * time.Second) for { select { case <-ctx.Done(): return case <-ticker.C: - if !iterInprogress { - iterInprogress = true - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) - readMarkers := make([]*ReadMarkerEntity, 0) - rm := &ReadMarkerEntity{RedeemRequired: true} - db.Where(rm). // redeem_required = true - Where("counter <> suspend"). // and not suspended - Order("created_at ASC").Find(&readMarkers) - if len(readMarkers) > 0 { - swg := sizedwaitgroup.New(config.Configuration.RMRedeemNumWorkers) - for _, rmEntity := range readMarkers { - swg.Add() - go func(redeemCtx context.Context, rmEntity *ReadMarkerEntity) { - redeemCtx = datastore.GetStore().CreateTransaction(redeemCtx) - defer redeemCtx.Done() - err := RedeemReadMarker(redeemCtx, rmEntity) - if err != nil { - Logger.Error("Error redeeming the read marker.", zap.Error(err)) - } - db := datastore.GetStore().GetTransaction(redeemCtx) - err = db.Commit().Error - if err != nil { - Logger.Error("Error commiting the readmarker redeem", zap.Error(err)) - } - swg.Done() - }(ctx, rmEntity) - } - swg.Wait() - } - db.Rollback() - rctx.Done() - iterInprogress = false - } + redeemReadMarker(ctx) } } diff --git a/code/go/0chain.net/blobbercore/reference/objectpath.go b/code/go/0chain.net/blobbercore/reference/objectpath.go index 3712b86e7..84d449131 100644 --- a/code/go/0chain.net/blobbercore/reference/objectpath.go +++ b/code/go/0chain.net/blobbercore/reference/objectpath.go @@ -12,6 +12,8 @@ type ObjectPath struct { Meta map[string]interface{} `json:"meta_data"` Path map[string]interface{} `json:"path"` FileBlockNum int64 `json:"file_block_num"` + ChunkSize int64 `json:"chunk_size"` + Size int64 `json:"size"` RefID int64 `json:"-"` } @@ -83,6 +85,8 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O retObj.Meta = curRef.GetListingData(ctx) retObj.Path = result retObj.FileBlockNum = remainingBlocks + retObj.ChunkSize = curRef.ChunkSize + retObj.Size = curRef.Size retObj.RefID = curRef.ID return &retObj, nil diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index f102a5cd5..ce3fa0272 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -90,6 +90,8 @@ type Ref struct { UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion + + ChunkSize int64 `gorm:"column:chunk_size" dirlist:"chunk_size" filelist:"chunk_size"` } type PaginatedRef struct { //Gorm smart select fields. @@ -123,12 +125,15 @@ type PaginatedRef struct { //Gorm smart select fields. CreatedAt time.Time `gorm:"column:created_at" json:"created_at,omitempty"` UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at,omitempty"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at" json:"-"` // soft deletion + + ChunkSize int64 `gorm:"column:chunk_size" dirlist:"chunk_size" filelist:"chunk_size"` } func (Ref) TableName() string { return "reference_objects" } +// GetReferenceLookup hash(allocationID + ":" + path) func GetReferenceLookup(allocationID string, path string) string { return encryption.Hash(allocationID + ":" + path) } @@ -166,6 +171,7 @@ func (r *Ref) SetAttributes(attr *Attributes) (err error) { return } +// GetReference get FileRef with allcationID and path from postgres func GetReference(ctx context.Context, allocationID string, path string) (*Ref, error) { ref := &Ref{} db := datastore.GetStore().GetTransaction(ctx) @@ -264,6 +270,7 @@ func (fr *Ref) GetFileHashData() string { hashArray = append(hashArray, strconv.FormatInt(fr.ActualFileSize, 10)) hashArray = append(hashArray, fr.ActualFileHash) hashArray = append(hashArray, string(fr.Attributes)) + hashArray = append(hashArray, strconv.FormatInt(fr.ChunkSize, 10)) return strings.Join(hashArray, ":") } @@ -272,7 +279,7 @@ func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool) (string, er // fmt.Println("Fileref hash data: " + fr.GetFileHashData()) fr.Hash = encryption.Hash(fr.GetFileHashData()) // fmt.Println("Fileref hash : " + fr.Hash) - fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / CHUNK_SIZE)) + fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize))) fr.PathHash = GetReferenceLookup(fr.AllocationID, fr.Path) fr.PathLevel = len(GetSubDirsFromPath(fr.Path)) + 1 //strings.Count(fr.Path, "/") fr.LookupHash = GetReferenceLookup(fr.AllocationID, fr.Path) @@ -284,6 +291,7 @@ func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool) (string, er } func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (string, error) { + // empty directory, return hash directly if len(r.Children) == 0 && !r.childrenLoaded { return r.Hash, nil } @@ -374,6 +382,7 @@ func (r *Ref) Save(ctx context.Context) error { return db.Save(r).Error } +// GetListingData reflect and convert all fields into map[string]interface{} func (r *Ref) GetListingData(ctx context.Context) map[string]interface{} { if r == nil { return make(map[string]interface{}) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 6da91b3d8..a0a81f209 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -21,6 +21,7 @@ func GetReferencePath(ctx context.Context, allocationID string, path string) (*R return GetReferencePathFromPaths(ctx, allocationID, []string{path}) } +// GetReferencePathFromPaths validate and build full dir tree from db, and CalculateHash and return root Ref func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths []string) (*Ref, error) { var refs []Ref db := datastore.GetStore().GetTransaction(ctx) @@ -41,11 +42,13 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths [ } } + // root reference_objects with parent_path="" db = db.Or("parent_path = ? AND allocation_id = ?", "", allocationID) err := db.Order("level, lookup_hash").Find(&refs).Error if err != nil { return nil, err } + // there is no any child reference_objects for affected path, and instert root reference_objects if len(refs) == 0 { return &Ref{Type: DIRECTORY, AllocationID: allocationID, Name: "/", Path: "/", ParentPath: "", PathLevel: 1}, nil } @@ -55,9 +58,11 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths [ return nil, common.NewError("invalid_dir_tree", "DB has invalid tree. Root not found in DB") } + // valdiate dir tree, and populate Ref's children for CalculateHash refMap := make(map[string]*Ref) refMap[rootRef.Path] = rootRef for i := 1; i < len(refs); i++ { + if _, ok := refMap[refs[i].ParentPath]; !ok { return nil, common.NewError("invalid_dir_tree", "DB has invalid tree.") } diff --git a/code/go/0chain.net/blobbercore/stats/handler.go b/code/go/0chain.net/blobbercore/stats/handler.go index bf1a1be0a..04614a126 100644 --- a/code/go/0chain.net/blobbercore/stats/handler.go +++ b/code/go/0chain.net/blobbercore/stats/handler.go @@ -6,10 +6,10 @@ import ( "html/template" "net/http" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/constants" "go.uber.org/zap" ) @@ -283,11 +283,11 @@ func StatsJSONHandler(ctx context.Context, r *http.Request) (interface{}, error) func GetStatsHandler(ctx context.Context, r *http.Request) (interface{}, error) { q := r.URL.Query() - ctx = context.WithValue(ctx, constants.ALLOCATION_CONTEXT_KEY, q.Get("allocation_id")) + ctx = context.WithValue(ctx, constants.ContextKeyAllocation, q.Get("allocation_id")) ctx = datastore.GetStore().CreateTransaction(ctx) db := datastore.GetStore().GetTransaction(ctx) defer db.Rollback() - allocationID := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationID := ctx.Value(constants.ContextKeyAllocation).(string) bs := &BlobberStats{} if len(allocationID) != 0 { // TODO: Get only the allocation info from DB diff --git a/code/go/0chain.net/blobbercore/writemarker/entity.go b/code/go/0chain.net/blobbercore/writemarker/entity.go index 27b7c5e4f..52e4b10bb 100644 --- a/code/go/0chain.net/blobbercore/writemarker/entity.go +++ b/code/go/0chain.net/blobbercore/writemarker/entity.go @@ -57,7 +57,7 @@ func (wm *WriteMarkerEntity) UpdateStatus(ctx context.Context, db := datastore.GetStore().GetTransaction(ctx) statusBytes, _ := json.Marshal(statusMessage) - fmt.Println(string(statusBytes)) + if status == Failed { wm.ReedeemRetries++ err = db.Model(wm).Updates(WriteMarkerEntity{ @@ -98,6 +98,7 @@ func (wm *WriteMarkerEntity) UpdateStatus(ctx context.Context, return } +// GetWriteMarkerEntity get WriteMarkerEntity from postgres func GetWriteMarkerEntity(ctx context.Context, allocation_root string) (*WriteMarkerEntity, error) { db := datastore.GetStore().GetTransaction(ctx) wm := &WriteMarkerEntity{} diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index d4e3a7e56..cdad31027 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -3,16 +3,17 @@ package writemarker import ( "context" "encoding/json" + "fmt" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "github.com/0chain/gosdk/constants" "go.uber.org/zap" ) @@ -45,15 +46,15 @@ func (wm *WriteMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.Al } if wm.WM.Size != co.Size { - return common.NewError("write_marker_validation_failed", "Write Marker size does not match the connection size") + return common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker size %v does not match the connection size %v", wm.WM.Size, co.Size)) } - clientPublicKey := ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) + clientPublicKey := ctx.Value(constants.ContextKeyClientKey).(string) if len(clientPublicKey) == 0 { return common.NewError("write_marker_validation_failed", "Could not get the public key of the client") } - clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) if len(clientID) == 0 || clientID != wm.WM.ClientID || clientID != co.ClientID || co.ClientID != wm.WM.ClientID { return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded") } diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index 84a112a13..09fb734f4 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -8,13 +8,12 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/remeh/sizedwaitgroup" "go.uber.org/zap" ) func SetupWorkers(ctx context.Context) { - go RedeemWriteMarkers(ctx) + go startRedeemWriteMarkers(ctx) } func RedeemMarkersForAllocation(ctx context.Context, allocationObj *allocation.Allocation) error { @@ -66,7 +65,7 @@ func RedeemMarkersForAllocation(ctx context.Context, allocationObj *allocation.A return nil } -func RedeemWriteMarkers(ctx context.Context) { +func startRedeemWriteMarkers(ctx context.Context) { var ticker = time.NewTicker( time.Duration(config.Configuration.WMRedeemFreq) * time.Second, ) @@ -77,27 +76,7 @@ func RedeemWriteMarkers(ctx context.Context) { case <-ticker.C: // Logger.Info("Trying to redeem writemarkers.", // zap.Any("numOfWorkers", numOfWorkers)) - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) - allocations := make([]*allocation.Allocation, 0) - alloc := &allocation.Allocation{IsRedeemRequired: true} - db.Where(alloc).Find(&allocations) - if len(allocations) > 0 { - swg := sizedwaitgroup.New(config.Configuration.WMRedeemNumWorkers) - for _, allocationObj := range allocations { - swg.Add() - go func(redeemCtx context.Context, allocationObj *allocation.Allocation) { - err := RedeemMarkersForAllocation(redeemCtx, allocationObj) - if err != nil { - Logger.Error("Error redeeming the write marker for allocation.", zap.Any("allocation", allocationObj.ID), zap.Error(err)) - } - swg.Done() - }(ctx, allocationObj) - } - swg.Wait() - } - db.Rollback() - rctx.Done() + redeemWriteMarker(ctx) } } diff --git a/code/go/0chain.net/blobbercore/writemarker/writemarker.go b/code/go/0chain.net/blobbercore/writemarker/writemarker.go new file mode 100644 index 000000000..99123d1e2 --- /dev/null +++ b/code/go/0chain.net/blobbercore/writemarker/writemarker.go @@ -0,0 +1,42 @@ +package writemarker + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/zap" +) + +func redeemWriteMarker(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + logging.Logger.Error("[recover] redeemWriteMarker", zap.Any("err", r)) + } + }() + + rctx := datastore.GetStore().CreateTransaction(ctx) + db := datastore.GetStore().GetTransaction(rctx) + allocations := make([]*allocation.Allocation, 0) + alloc := &allocation.Allocation{IsRedeemRequired: true} + db.Where(alloc).Find(&allocations) + if len(allocations) > 0 { + swg := sizedwaitgroup.New(config.Configuration.WMRedeemNumWorkers) + for _, allocationObj := range allocations { + swg.Add() + go func(redeemCtx context.Context, allocationObj *allocation.Allocation) { + err := RedeemMarkersForAllocation(redeemCtx, allocationObj) + if err != nil { + logging.Logger.Error("Error redeeming the write marker for allocation.", zap.Any("allocation", allocationObj.ID), zap.Error(err)) + } + swg.Done() + }(ctx, allocationObj) + } + swg.Wait() + } + db.Rollback() + rctx.Done() +} diff --git a/code/go/0chain.net/core/common/types.go b/code/go/0chain.net/core/common/types.go index a5c79cda5..73448f77e 100644 --- a/code/go/0chain.net/core/common/types.go +++ b/code/go/0chain.net/core/common/types.go @@ -4,9 +4,6 @@ import ( "fmt" ) -/*ContextKey - type for key used to store values into context */ -type ContextKey string - // WhoPays for file downloading. type WhoPays int diff --git a/code/go/0chain.net/core/node/context.go b/code/go/0chain.net/core/node/context.go index 120c160cd..083851263 100644 --- a/code/go/0chain.net/core/node/context.go +++ b/code/go/0chain.net/core/node/context.go @@ -3,10 +3,10 @@ package node import ( "context" - "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/gosdk/constants" ) -const SELF_NODE common.ContextKey = "SELF_NODE" +const SELF_NODE constants.ContextKey = "SELF_NODE" /*GetNodeContext - setup a context with the self node */ func GetNodeContext() context.Context { diff --git a/code/go/0chain.net/core/util/merkle_tree.go b/code/go/0chain.net/core/util/merkle_tree.go deleted file mode 100644 index 675b05df2..000000000 --- a/code/go/0chain.net/core/util/merkle_tree.go +++ /dev/null @@ -1,148 +0,0 @@ -package util - -import ( - "fmt" -) - -/*MerkleTree - A data structure that implements MerkleTreeI interface */ -type MerkleTree struct { - tree []string - leavesCount int - levels int -} - -func VerifyMerklePath(hash string, path *MTPath, root string) bool { - mthash := hash - pathNodes := path.Nodes - pl := len(pathNodes) - idx := path.LeafIndex - for i := 0; i < pl; i++ { - if idx&1 == 1 { - mthash = MHash(pathNodes[i], mthash) - } else { - mthash = MHash(mthash, pathNodes[i]) - } - idx = (idx - idx&1) / 2 - } - return mthash == root -} - -func (mt *MerkleTree) computeSize(leaves int) (int, int) { - if leaves == 1 { - return 2, 2 - } - var tsize int - var levels int - for ll := leaves; ll > 1; ll = (ll + 1) / 2 { - tsize += ll - levels++ - } - tsize++ - levels++ - return tsize, levels -} - -/*ComputeTree - given the leaf nodes, compute the merkle tree */ -func (mt *MerkleTree) ComputeTree(hashes []Hashable) { - var tsize int - tsize, mt.levels = mt.computeSize(len(hashes)) - mt.leavesCount = len(hashes) - mt.tree = make([]string, tsize) - for idx, hashable := range hashes { - mt.tree[idx] = hashable.GetHash() - } - if len(hashes) == 1 { - mt.tree[1] = MHash(mt.tree[0], mt.tree[0]) - return - } - for pl0, plsize := 0, mt.leavesCount; plsize > 1; pl0, plsize = pl0+plsize, (plsize+1)/2 { - l0 := pl0 + plsize - for i, j := 0, 0; i < plsize; i, j = i+2, j+1 { - mt.tree[pl0+plsize+j] = MHash(mt.tree[pl0+i], mt.tree[pl0+i+1]) - } - if plsize&1 == 1 { - mt.tree[l0+plsize/2] = MHash(mt.tree[pl0+plsize-1], mt.tree[pl0+plsize-1]) - } - } -} - -/*GetRoot - get the root of the merkle tree */ -func (mt *MerkleTree) GetRoot() string { - return mt.tree[len(mt.tree)-1] -} - -/*GetTree - get the entire merkle tree */ -func (mt *MerkleTree) GetTree() []string { - return mt.tree -} - -/*SetTree - set the entire merkle tree */ -func (mt *MerkleTree) SetTree(leavesCount int, tree []string) error { - size, levels := mt.computeSize(leavesCount) - if size != len(tree) { - return fmt.Errorf("Merkle tree with leaves %v should have size %v but only %v is given", leavesCount, size, len(tree)) - } - mt.levels = levels - mt.tree = tree - mt.leavesCount = leavesCount - return nil -} - -/*GetLeafIndex - Get the index of the leaf node in the tree */ -func (mt *MerkleTree) GetLeafIndex(hash Hashable) int { - hs := hash.GetHash() - for i := 0; i < mt.leavesCount; i++ { - if mt.tree[i] == hs { - return i - } - } - return -1 -} - -/*GetPath - get the path that can be used to verify the merkle tree */ -func (mt *MerkleTree) GetPath(hash Hashable) *MTPath { - hidx := mt.GetLeafIndex(hash) - if hidx < 0 { - return &MTPath{} - } - return mt.GetPathByIndex(hidx) -} - -/*VerifyPath - given a leaf node and the path, verify that the node is part of the tree */ -func (mt *MerkleTree) VerifyPath(hash Hashable, path *MTPath) bool { - hs := hash.GetHash() - return VerifyMerklePath(hs, path, mt.GetRoot()) -} - -/*GetPathByIndex - get the path of a leaf node at index i */ -func (mt *MerkleTree) GetPathByIndex(idx int) *MTPath { - path := make([]string, mt.levels-1, mt.levels-1) //nolint:gosimple // need more time to verify: declaring capacity is probably necessary? - mpath := &MTPath{LeafIndex: idx} - if idx&1 == 1 { - path[0] = mt.tree[idx-1] - } else { - if idx+1 < mt.leavesCount { - path[0] = mt.tree[idx+1] - } else { - path[0] = mt.tree[idx] - } - } - for pl0, plsize, pi := 0, mt.leavesCount, 1; plsize > 2; pl0, plsize, pi = pl0+plsize, (plsize+1)/2, pi+1 { - l0 := pl0 + plsize - idx = (idx - idx&1) / 2 - if idx&1 == 1 { - //path = append(path, mt.tree[l0+idx-1]) - path[pi] = mt.tree[l0+idx-1] - } else { - if l0+idx+1 < l0+(plsize+1)/2 { - //path = append(path, mt.tree[l0+idx+1]) - path[pi] = mt.tree[l0+idx+1] - } else { - //path = append(path, mt.tree[l0+idx]) - path[pi] = mt.tree[l0+idx] - } - } - } - mpath.Nodes = path - return mpath -} diff --git a/code/go/0chain.net/core/util/merkle_tree_interface.go b/code/go/0chain.net/core/util/merkle_tree_interface.go deleted file mode 100644 index 4eeea3ede..000000000 --- a/code/go/0chain.net/core/util/merkle_tree_interface.go +++ /dev/null @@ -1,53 +0,0 @@ -package util - -import "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - -/*MerkleTreeI - a merkle tree interface required for constructing and providing verification */ -type MerkleTreeI interface { - //API to create a tree from leaf nodes - ComputeTree(hashes []Hashable) - GetRoot() string - GetTree() []string - - //API to load an existing tree - SetTree(leavesCount int, tree []string) error - - // API for verification when the leaf node is known - GetPath(hash Hashable) *MTPath // Server needs to provide this - VerifyPath(hash Hashable, path *MTPath) bool //This is only required by a client but useful for testing - - /* API for random verification when the leaf node is uknown - (verification of the data to hash used as leaf node is outside this API) */ - GetPathByIndex(idx int) *MTPath -} - -/*MTPath - The merkle tree path*/ -type MTPath struct { - Nodes []string `json:"nodes"` - LeafIndex int `json:"leaf_index"` -} - -/*Hash - the hashing used for the merkle tree construction */ -func Hash(text string) string { - return encryption.Hash(text) -} - -/*MHash - merkle hashing of a pair of child hashes */ -func MHash(h1 string, h2 string) string { - return Hash(h1 + h2) -} - -type StringHashable struct { - Hash string -} - -func NewStringHashable(hash string) *StringHashable { - return &StringHashable{Hash: hash} -} - -func (sh *StringHashable) GetHash() string { - return sh.Hash -} -func (sh *StringHashable) GetHashBytes() []byte { - return []byte(sh.Hash) -} diff --git a/code/go/0chain.net/core/util/merkle_tree_test.go b/code/go/0chain.net/core/util/merkle_tree_test.go deleted file mode 100644 index 57fe65380..000000000 --- a/code/go/0chain.net/core/util/merkle_tree_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package util - -import ( - "fmt" - "math/rand" - "testing" - - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" -) - -type Txn struct { - data string -} - -func (t *Txn) GetHash() string { - return t.data -} - -func (t *Txn) GetHashBytes() []byte { - return encryption.RawHash(t.data) -} - -func (t *Txn) Encode() []byte { - return []byte(t.data) -} - -func (t *Txn) Decode(data []byte) error { - t.data = string(data) - return nil -} - -func TestMerkleTreeComputeTree(t *testing.T) { - txns := make([]Hashable, 100) - for i := 0; i < len(txns); i++ { - txns[i] = &Txn{data: fmt.Sprintf("%v", 1001-i)} - } - var mt MerkleTreeI = &MerkleTree{} - mt.ComputeTree(txns) - tree := mt.GetTree() - if len(tree) != 202 { - fmt.Printf("%v: %v\n", len(tree), tree) - } -} - -func TestMerkleTreeGetNVerifyPath(t *testing.T) { - txns := make([]Hashable, 101) - for i := 0; i < len(txns); i++ { - txns[i] = &Txn{data: fmt.Sprintf("1000%v", i)} - } - var mt MerkleTreeI = &MerkleTree{} - mt.ComputeTree(txns) - for i := 0; i < len(txns); i++ { - path := mt.GetPath(txns[i]) - if !mt.VerifyPath(txns[i], path) { - fmt.Printf("path: %v %v\n", txns[i], path) - } - } -} - -func TestMerkleTreeSetTree(t *testing.T) { - txns := make([]Hashable, 100) - for i := 0; i < len(txns); i++ { - txns[i] = &Txn{data: fmt.Sprintf("%v", 1001-i)} - } - var mt MerkleTreeI = &MerkleTree{} - mt.ComputeTree(txns) - var mt2 MerkleTreeI = &MerkleTree{} - err := mt2.SetTree(len(txns), mt.GetTree()) - if err != nil { - t.Error(err) - } - if mt.GetRoot() != mt2.GetRoot() { - t.Errorf("Merkle roots didn't match") - } -} - -func BenchmarkMerkleTreeComputeTree(b *testing.B) { - txns := make([]Hashable, 10000) - for i := 0; i < len(txns); i++ { - txns[i] = &Txn{data: fmt.Sprintf("%v", len(txns)-i)} - } - for i := 0; i < b.N; i++ { - var mt MerkleTreeI = &MerkleTree{} - mt.ComputeTree(txns) - } -} - -func BenchmarkMerkleTreeGetPath(b *testing.B) { - txns := make([]Hashable, 10000) - for i := 0; i < len(txns); i++ { - txns[i] = &Txn{data: fmt.Sprintf("%v", len(txns)-i)} - } - var mt MerkleTreeI = &MerkleTree{} - mt.ComputeTree(txns) - for i := 0; i < b.N; i++ { - j := rand.Intn(len(txns)) - mt.GetPath(txns[j]) - } -} - -func BenchmarkMerkleTreeVerifyPath(b *testing.B) { - txns := make([]Hashable, 10000) - for i := 0; i < len(txns); i++ { - txns[i] = &Txn{data: fmt.Sprintf("%v", len(txns)-i)} - } - var mt MerkleTreeI = &MerkleTree{} - mt.ComputeTree(txns) - paths := make([]*MTPath, len(txns)) - for j := 0; j < len(txns); j++ { - paths[j] = mt.GetPath(txns[j]) - } - for i := 0; i < b.N; i++ { - j := rand.Intn(len(txns)) - - if !mt.VerifyPath(txns[j], paths[j]) { - fmt.Printf("path verification failed") - return - } - } -} diff --git a/code/go/0chain.net/dev/miner/init.go b/code/go/0chain.net/dev/miner/init.go new file mode 100644 index 000000000..5bd64dbb8 --- /dev/null +++ b/code/go/0chain.net/dev/miner/init.go @@ -0,0 +1,11 @@ +package miner + +import ( + "github.com/gorilla/mux" +) + +func RegisterHandlers(s *mux.Router) { + // s.HandleFunc("/v1/file/upload/{allocation}", uploadAndUpdateFile).Methods(http.MethodPut, http.MethodPost) + // s.HandleFunc("/v1/file/referencepath/{allocation}", getReference).Methods(http.MethodGet) + // s.HandleFunc("/v1/connection/commit/{allocation}", commitWrite).Methods(http.MethodPost) +} diff --git a/code/go/0chain.net/dev/server.go b/code/go/0chain.net/dev/server.go new file mode 100644 index 000000000..0c213562a --- /dev/null +++ b/code/go/0chain.net/dev/server.go @@ -0,0 +1,45 @@ +// package dev providers tools for local development +package dev + +import ( + "net/http/httptest" + + "github.com/0chain/blobber/code/go/0chain.net/dev/miner" + "github.com/0chain/blobber/code/go/0chain.net/dev/sharder" + "github.com/gorilla/mux" +) + +// Server a local dev server to mock server APIs +type Server struct { + *httptest.Server + *mux.Router +} + +// NewServer create a local dev server +func NewServer() *Server { + router := mux.NewRouter() + s := &Server{ + Router: router, + Server: httptest.NewServer(router), + } + + return s +} + +// NewSharderServer create a local dev sharder server +func NewSharderServer() *Server { + s := NewServer() + + sharder.RegisterHandlers(s.Router) + + return s +} + +// NewMinerServer create a local dev miner server +func NewMinerServer() *Server { + s := NewServer() + + miner.RegisterHandlers(s.Router) + + return s +} diff --git a/code/go/0chain.net/dev/sharder/init.go b/code/go/0chain.net/dev/sharder/init.go new file mode 100644 index 000000000..f6992d3cc --- /dev/null +++ b/code/go/0chain.net/dev/sharder/init.go @@ -0,0 +1,11 @@ +package sharder + +import ( + "github.com/gorilla/mux" +) + +func RegisterHandlers(s *mux.Router) { + // s.HandleFunc("/v1/file/upload/{allocation}", uploadAndUpdateFile).Methods(http.MethodPut, http.MethodPost) + // s.HandleFunc("/v1/file/referencepath/{allocation}", getReference).Methods(http.MethodGet) + // s.HandleFunc("/v1/connection/commit/{allocation}", commitWrite).Methods(http.MethodPost) +} diff --git a/code/go/0chain.net/validator/main.go b/code/go/0chain.net/validator/main.go index 4d673d13e..7a1b3e0c0 100644 --- a/code/go/0chain.net/validator/main.go +++ b/code/go/0chain.net/validator/main.go @@ -44,11 +44,12 @@ func main() { logDir := flag.String("log_dir", "", "log_dir") portString := flag.String("port", "", "port") hostname := flag.String("hostname", "", "hostname") + configDir := flag.String("config_dir", "./config", "config_dir") flag.Parse() config.SetupDefaultConfig() - config.SetupConfig() + config.SetupConfig(*configDir) config.Configuration.DeploymentMode = byte(*deploymentMode) diff --git a/code/go/0chain.net/validatorcore/config/config.go b/code/go/0chain.net/validatorcore/config/config.go index f6ae4ce8f..4b59c4bd7 100644 --- a/code/go/0chain.net/validatorcore/config/config.go +++ b/code/go/0chain.net/validatorcore/config/config.go @@ -17,9 +17,14 @@ func SetupDefaultConfig() { } /*SetupConfig - setup the configuration system */ -func SetupConfig() { +func SetupConfig(configDir string) { viper.SetConfigName("0chain_validator") - viper.AddConfigPath("./config") + if configDir == "" { + viper.AddConfigPath("./config") + } else { + viper.AddConfigPath(configDir) + } + err := viper.ReadInConfig() // Find and read the config file if err != nil { // Handle errors reading the config file panic(fmt.Errorf("fatal error config file: %s", err)) diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler.go b/code/go/0chain.net/validatorcore/storage/challenge_handler.go index ea3a78d01..28add0fd9 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler.go @@ -87,6 +87,7 @@ func ChallengeHandler(ctx context.Context, r *http.Request) (interface{}, error) validationTicket.ValidatorID = node.Self.ID validationTicket.ValidatorKey = node.Self.PublicKey validationTicket.Timestamp = common.Now() + if err := validationTicket.Sign(); err != nil { return nil, common.NewError("invalid_parameters", err.Error()) } diff --git a/code/go/0chain.net/validatorcore/storage/context.go b/code/go/0chain.net/validatorcore/storage/context.go index 5cfa922c8..86a2acb5d 100644 --- a/code/go/0chain.net/validatorcore/storage/context.go +++ b/code/go/0chain.net/validatorcore/storage/context.go @@ -5,12 +5,13 @@ import ( "net/http" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/gosdk/constants" ) func SetupContext(handler common.JSONResponderF) common.JSONResponderF { return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = context.WithValue(ctx, CLIENT_CONTEXT_KEY, r.Header.Get(common.ClientHeader)) - ctx = context.WithValue(ctx, CLIENT_KEY_CONTEXT_KEY, + ctx = context.WithValue(ctx, constants.ContextKeyClient, r.Header.Get(common.ClientHeader)) + ctx = context.WithValue(ctx, constants.ContextKeyClientKey, r.Header.Get(common.ClientKeyHeader)) res, err := handler(ctx, r) return res, err diff --git a/code/go/0chain.net/validatorcore/storage/models.go b/code/go/0chain.net/validatorcore/storage/models.go index b64296932..a5f93d0fb 100644 --- a/code/go/0chain.net/validatorcore/storage/models.go +++ b/code/go/0chain.net/validatorcore/storage/models.go @@ -1,6 +1,7 @@ package storage import ( + "bytes" "encoding/json" "fmt" "math/rand" @@ -11,8 +12,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/blobber/code/go/0chain.net/core/util" "github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage/writemarker" + "github.com/0chain/gosdk/core/util" "github.com/mitchellh/mapstructure" @@ -91,6 +92,7 @@ type FileMetaData struct { MerkleRoot string `json:"merkle_root" mapstructure:"merkle_root"` ActualFileSize int64 `json:"actual_file_size" mapstructure:"actual_file_size"` ActualFileHash string `json:"actual_file_hash" mapstructure:"actual_file_hash"` + ChunkSize int64 `json:"chunk_size" mapstructure:"chunk_size"` Attributes Attributes `json:"attributes" mapstructure:"attributes" ` } @@ -106,6 +108,7 @@ func (fr *FileMetaData) GetHashData() string { hashArray = append(hashArray, strconv.FormatInt(fr.ActualFileSize, 10)) hashArray = append(hashArray, fr.ActualFileHash) hashArray = append(hashArray, fr.Attributes.String()) + hashArray = append(hashArray, strconv.FormatInt(fr.ChunkSize, 10)) return strings.Join(hashArray, ":") } @@ -286,6 +289,7 @@ type ChallengeRequest struct { WriteMarkers []*writemarker.WriteMarkerEntity `json:"write_markers,omitempty"` DataBlock []byte `json:"data,omitempty"` MerklePath *util.MTPath `json:"merkle_path,omitempty"` + ChunkSize int64 `json:"chunk_size,omitempty"` } func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationObj *Allocation) error { @@ -326,8 +330,15 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO } Logger.Info("Verifying data block and merkle path", zap.Any("challenge_id", challengeObj.ID)) - contentHash := encryption.Hash(cr.DataBlock) - merkleVerify := util.VerifyMerklePath(contentHash, cr.MerklePath, cr.ObjPath.Meta.MerkleRoot) + //contentHash := encryption.Hash(cr.DataBlock) + + contentHasher := util.NewCompactMerkleTree(nil) + err = contentHasher.Reload(cr.ChunkSize, bytes.NewReader(cr.DataBlock)) + if err != nil { + return common.NewError("challenge_validation_failed", "Failed to calculate content hash for the data block") + } + + merkleVerify := util.VerifyMerklePath(contentHasher.GetMerkleRoot(), cr.MerklePath, cr.ObjPath.Meta.MerkleRoot) if !merkleVerify { return common.NewError("challenge_validation_failed", "Failed to verify the merkle path for the data block") } diff --git a/code/go/0chain.net/validatorcore/storage/models_test.go b/code/go/0chain.net/validatorcore/storage/models_test.go index c684c0d9a..8b56b577e 100644 --- a/code/go/0chain.net/validatorcore/storage/models_test.go +++ b/code/go/0chain.net/validatorcore/storage/models_test.go @@ -141,7 +141,7 @@ func TestFileMetaData_GetHashData(t *testing.T) { WhoPaysForReads: common.WhoPaysOwner, }, }, - want: "::::0:::0::{}", + want: "::::0:::0::{}:0", // want: "::::0:::0::{\"who_pays_for_reads\":0}", }, { @@ -152,14 +152,14 @@ func TestFileMetaData_GetHashData(t *testing.T) { WhoPaysForReads: common.WhoPays3rdParty, }, }, - want: "::::0:::0::{\"who_pays_for_reads\":1}", + want: "::::0:::0::{\"who_pays_for_reads\":1}:0", }, { name: "with Attributes.WhoPays = nil", fmd: storage.FileMetaData{ DirMetaData: storage.DirMetaData{}, }, - want: "::::0:::0::{}", + want: "::::0:::0::{}:0", }, } @@ -192,7 +192,7 @@ func TestFileMetaData_CalculateHash(t *testing.T) { WhoPaysForReads: common.WhoPaysOwner, }, }, - want: "f78718c8ad33d8b97fe902dabc36df401f82c88bde608ab85005d332ac24de43", + want: "2f3e218c5c0a759f8267f8b86d8438c5416c1c8499cd483f1fae428fe97eca2d", // want: "a9862b25db264157a540dc3ecf7aae331f377aeb3c2ef9c59951e1c8c3e3bc15", }, { @@ -202,12 +202,12 @@ func TestFileMetaData_CalculateHash(t *testing.T) { WhoPaysForReads: common.WhoPays3rdParty, }, }, - want: "2cc4e60833e5bf1e018910ce256d12374f1d8e87beade0f5c2a63770e6b8a445", + want: "b5553813cb69c0973b25ba19d9fc7748bd8eb4cc3ee7925f55750c96b08c5804", }, { name: "with Attributes.WhoPays = nil", fmd: storage.FileMetaData{}, - want: "f78718c8ad33d8b97fe902dabc36df401f82c88bde608ab85005d332ac24de43", + want: "2f3e218c5c0a759f8267f8b86d8438c5416c1c8499cd483f1fae428fe97eca2d", }, } @@ -269,7 +269,7 @@ func TestObjectPath_Parse(t *testing.T) { "list": []map[string]interface{}{ map[string]interface{}{ "path": "file.txt", - "hash": "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", + "hash": "7f0823cc8a8a4bcf22335e87e7e629eda04b6517b00c2db4049b125ffb634cb1", "type": "f", }, }, @@ -443,15 +443,15 @@ func TestObjectPath_VerifyPath(t *testing.T) { { name: "dir/file", objPath: &storage.ObjectPath{ - RootHash: "b25a7f67d4206d77fca08a48a06eba893c59077ea61435f71b31d098ea2f7991", + RootHash: "8bbf60d57fa672bbd6bcc49926148afd39930dff3a54722a905c3f12e150e29e", Path: map[string]interface{}{ "path": "dir1", - "hash": "b25a7f67d4206d77fca08a48a06eba893c59077ea61435f71b31d098ea2f7991", + "hash": "8bbf60d57fa672bbd6bcc49926148afd39930dff3a54722a905c3f12e150e29e", "type": "d", "list": []map[string]interface{}{ map[string]interface{}{ "path": "file.txt", - "hash": "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", + "hash": "7f0823cc8a8a4bcf22335e87e7e629eda04b6517b00c2db4049b125ffb634cb1", "type": "f", }, }, @@ -461,7 +461,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { Type: storage.DIRECTORY, Name: "", Path: "dir1", - Hash: "b25a7f67d4206d77fca08a48a06eba893c59077ea61435f71b31d098ea2f7991", + Hash: "8bbf60d57fa672bbd6bcc49926148afd39930dff3a54722a905c3f12e150e29e", PathHash: "", NumBlocks: int64(0), AllocationID: "", @@ -472,7 +472,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { Type: storage.FILE, Name: "", Path: "file.txt", - Hash: "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", + Hash: "7f0823cc8a8a4bcf22335e87e7e629eda04b6517b00c2db4049b125ffb634cb1", PathHash: "", NumBlocks: int64(0), AllocationID: "1", @@ -497,12 +497,12 @@ func TestObjectPath_VerifyPath(t *testing.T) { RootHash: "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", Path: map[string]interface{}{ "path": "dir1", - "hash": "b25a7f67d4206d77fca08a48a06eba893c59077ea61435f71b31d098ea2f7991", + "hash": "8bbf60d57fa672bbd6bcc49926148afd39930dff3a54722a905c3f12e150e29e", "type": "d", "list": []map[string]interface{}{ map[string]interface{}{ "path": "file.txt", - "hash": "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", + "hash": "7f0823cc8a8a4bcf22335e87e7e629eda04b6517b00c2db4049b125ffb634cb1", "type": "f", }, }, @@ -512,7 +512,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { Type: storage.DIRECTORY, Name: "", Path: "dir1", - Hash: "b25a7f67d4206d77fca08a48a06eba893c59077ea61435f71b31d098ea2f7991", + Hash: "8bbf60d57fa672bbd6bcc49926148afd39930dff3a54722a905c3f12e150e29e", PathHash: "", NumBlocks: int64(0), AllocationID: "", @@ -523,7 +523,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { Type: storage.FILE, Name: "", Path: "file.txt", - Hash: "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", + Hash: "7f0823cc8a8a4bcf22335e87e7e629eda04b6517b00c2db4049b125ffb634cb1", PathHash: "", NumBlocks: int64(0), AllocationID: "1", @@ -547,20 +547,20 @@ func TestObjectPath_VerifyPath(t *testing.T) { { name: "dir/dir/file", objPath: &storage.ObjectPath{ - RootHash: "a02b02080606e78e165fe5a42f8b0087ff82617a1f9c26cc95e269fd653c5a72", + RootHash: "eb5167bd68f64f8b1dfae1f14230b43601bd1d42060dd1c2ec69a3674828018a", Path: map[string]interface{}{ "path": "dir1", - "hash": "a02b02080606e78e165fe5a42f8b0087ff82617a1f9c26cc95e269fd653c5a72", + "hash": "eb5167bd68f64f8b1dfae1f14230b43601bd1d42060dd1c2ec69a3674828018a", "type": "d", "list": []map[string]interface{}{ map[string]interface{}{ "path": "dir2", - "hash": "b25a7f67d4206d77fca08a48a06eba893c59077ea61435f71b31d098ea2f7991", + "hash": "8bbf60d57fa672bbd6bcc49926148afd39930dff3a54722a905c3f12e150e29e", "type": "d", "list": []map[string]interface{}{ map[string]interface{}{ "path": "file.txt", - "hash": "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", + "hash": "7f0823cc8a8a4bcf22335e87e7e629eda04b6517b00c2db4049b125ffb634cb1", "type": "f", }, }, @@ -572,7 +572,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { Type: storage.DIRECTORY, Name: "", Path: "dir1", - Hash: "a02b02080606e78e165fe5a42f8b0087ff82617a1f9c26cc95e269fd653c5a72", + Hash: "eb5167bd68f64f8b1dfae1f14230b43601bd1d42060dd1c2ec69a3674828018a", PathHash: "", NumBlocks: int64(0), AllocationID: "", @@ -582,7 +582,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { Type: storage.DIRECTORY, Name: "", Path: "dir2", - Hash: "b25a7f67d4206d77fca08a48a06eba893c59077ea61435f71b31d098ea2f7991", + Hash: "8bbf60d57fa672bbd6bcc49926148afd39930dff3a54722a905c3f12e150e29e", PathHash: "", NumBlocks: int64(0), AllocationID: "1", @@ -593,7 +593,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { Type: storage.FILE, Name: "", Path: "file.txt", - Hash: "87177591985fdf5c010d7781f0dc82b5d3c40b6bf8892b3c69000eb000f1e33a", + Hash: "7f0823cc8a8a4bcf22335e87e7e629eda04b6517b00c2db4049b125ffb634cb1", PathHash: "", NumBlocks: int64(0), AllocationID: "1", diff --git a/code/go/0chain.net/validatorcore/storage/protocol.go b/code/go/0chain.net/validatorcore/storage/protocol.go index c58319887..192027a3d 100644 --- a/code/go/0chain.net/validatorcore/storage/protocol.go +++ b/code/go/0chain.net/validatorcore/storage/protocol.go @@ -13,14 +13,12 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "github.com/0chain/blobber/code/go/0chain.net/validatorcore/config" + "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zcncore" "go.uber.org/zap" ) const CHUNK_SIZE = 64 * 1024 -const ALLOCATION_CONTEXT_KEY common.ContextKey = "allocation" -const CLIENT_CONTEXT_KEY common.ContextKey = "client" -const CLIENT_KEY_CONTEXT_KEY common.ContextKey = "client_key" type StorageNode struct { ID string `json:"id"` @@ -84,7 +82,7 @@ func (sp *ValidatorProtocolImpl) VerifyAllocationTransaction(ctx context.Context } func (sp *ValidatorProtocolImpl) VerifyChallengeTransaction(ctx context.Context, challengeRequest *ChallengeRequest) (*Challenge, error) { - blobberID := ctx.Value(CLIENT_CONTEXT_KEY).(string) + blobberID := ctx.Value(constants.ContextKeyClient).(string) if len(blobberID) == 0 { return nil, common.NewError("invalid_client", "Call from an invalid client") } diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index 155ca445f..2eb5bce8e 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -1,8 +1,8 @@ version: 1.0 logging: - level: "info" - console: false # printing log to console is only supported in development mode + level: "error" + console: true # printing log to console is only supported in development mode # for testing # 500 MB - 536870912 @@ -36,7 +36,7 @@ challenge_completion_time: 2m # duration to complete a challenge # a task and redeem tokens, it should be big enough read_lock_timeout: 1m write_lock_timeout: 1m -max_file_size: 10485760 #10MB +max_file_size: 104857600 #10MB # update_allocations_interval used to refresh known allocation objects from SC update_allocations_interval: 1m @@ -59,7 +59,7 @@ min_confirmation: 50 block_worker: http://198.18.0.98:9091 handlers: - rate_limit: 10 # 10 per second + rate_limit: 0 # 10 per second . it can't too small one if a large file is download with blocks server_chain: id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" @@ -73,7 +73,7 @@ contentref_cleaner: tolerance: 3600 openconnection_cleaner: frequency: 30 - tolerance: 3600 + tolerance: 3600 # 60 * 60 writemarker_redeem: frequency: 10 num_workers: 5 diff --git a/config/0chain_validator.yaml b/config/0chain_validator.yaml index 0173a38c7..af24b28ba 100644 --- a/config/0chain_validator.yaml +++ b/config/0chain_validator.yaml @@ -17,8 +17,8 @@ handlers: rate_limit: 10 # 10 per second logging: - level: "info" - console: false # printing log to console is only supported in development mode + level: "error" + console: true # printing log to console is only supported in development mode server_chain: id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" diff --git a/dev.local/README.md b/dev.local/README.md new file mode 100644 index 000000000..7662b4cf3 --- /dev/null +++ b/dev.local/README.md @@ -0,0 +1,86 @@ +# blobber development guide + + + + +## install `postgres:11` in docker as shared database for blobbers and validators + +``` +********************************************** + Welcome to blobber/validator development CLI +********************************************** + + +Please select which blobber/validator you will work on: +1) 1 +2) 2 +3) 3 +4) clean all +#? 1 + +********************************************** + Blobber/Validator 1 +********************************************** + +Please select what you will do: +1) install postgres +2) start blobber +3) start validator +#? 1 +``` + + +It will check and install a `blobber_postgres` container as shared database for all blobbers and validators, and initialized database `blobber_meta$i` and user `blobber_user$i` + +## run local validator instance + +``` +********************************************** + Welcome to blobber/validator development CLI +********************************************** + + +Please select which blobber/validator you will work on: +1) 1 +2) 2 +3) 3 +4) clean all +#? 1 + +********************************************** + Blobber/Validator 1 +********************************************** + +Please select what you will do: +1) install postgres 3) start validator +2) start blobber 4) clean +#? 3 + +``` + + +## run local blobber instance + +``` +********************************************** + Welcome to blobber/validator development CLI +********************************************** + + +Please select which blobber/validator you will work on: +1) 1 +2) 2 +3) 3 +4) clean all +#? 1 + +********************************************** + Blobber/Validator 1 +********************************************** + +Please select what you will do: +1) install postgres 3) start validator +2) start blobber 4) clean +#? 2 + +``` \ No newline at end of file diff --git a/dev.local/cli.sh b/dev.local/cli.sh new file mode 100755 index 000000000..06216cecb --- /dev/null +++ b/dev.local/cli.sh @@ -0,0 +1,249 @@ +#!/bin/bash + +set -e + +root=$(pwd) +hostname=`ifconfig | grep "inet " | grep -Fv 127.0.0.1 | grep broadcast | awk '{print $2; exit}'` + + +ips=`ifconfig | grep "inet " | grep 198.18.0 | wc -l` + + +#fix docker network issue for Mac OS X platform +if [ "$(uname)" == "Darwin" ] && [ $ips != 31 ] +then + # 0dns + sudo ifconfig lo0 alias 198.18.0.98 + # sharders + sudo ifconfig lo0 alias 198.18.0.81 + sudo ifconfig lo0 alias 198.18.0.82 + sudo ifconfig lo0 alias 198.18.0.83 + sudo ifconfig lo0 alias 198.18.0.84 + sudo ifconfig lo0 alias 198.18.0.85 + sudo ifconfig lo0 alias 198.18.0.86 + sudo ifconfig lo0 alias 198.18.0.87 + sudo ifconfig lo0 alias 198.18.0.88 + # miners + sudo ifconfig lo0 alias 198.18.0.71 + sudo ifconfig lo0 alias 198.18.0.72 + sudo ifconfig lo0 alias 198.18.0.73 + sudo ifconfig lo0 alias 198.18.0.74 + sudo ifconfig lo0 alias 198.18.0.75 + sudo ifconfig lo0 alias 198.18.0.76 + sudo ifconfig lo0 alias 198.18.0.77 + sudo ifconfig lo0 alias 198.18.0.78 + # blobbers + sudo ifconfig lo0 alias 198.18.0.91 + sudo ifconfig lo0 alias 198.18.0.92 + sudo ifconfig lo0 alias 198.18.0.93 + sudo ifconfig lo0 alias 198.18.0.94 + sudo ifconfig lo0 alias 198.18.0.95 + sudo ifconfig lo0 alias 198.18.0.96 + sudo ifconfig lo0 alias 198.18.0.97 + # validators + sudo ifconfig lo0 alias 198.18.0.61 + sudo ifconfig lo0 alias 198.18.0.62 + sudo ifconfig lo0 alias 198.18.0.63 + sudo ifconfig lo0 alias 198.18.0.64 + sudo ifconfig lo0 alias 198.18.0.65 + sudo ifconfig lo0 alias 198.18.0.66 + sudo ifconfig lo0 alias 198.18.0.67 +fi + + +echo " +********************************************** + Welcome to blobber/validator development CLI +********************************************** + +" + +echo "Hostname: $hostname" + +set_hostname() { + + read -p "change hostname($hostname), please enter your hostname: " hostname + echo "" + echo "> hostname is updated to: $hostname" +} + +change_zcn() { + zcn=$(cat ../config/0chain_blobber.yaml | grep '^block_worker' | awk -F ' ' '{print $2}') + read -p "change zcn($zcn), please enter your zcn(leave blank for skip): " yourZCN + + if [ ! -z "$yourZCN" -a "$yourZCN" != " " ]; then + find ../config/ -name "0chain_blobber.yaml" -exec sed -i '' "s/block_worker/#block_worker/g" {} \; + find ../config/ -name "0chain_validator.yaml" -exec sed -i '' "s/block_worker/#block_worker/g" {} \; + echo "block_worker: $yourZCN" >> ../config/0chain_blobber.yaml + echo "block_worker: $yourZCN" >> ../config/0chain_validator.yaml + fi + zcn=$(cat ../config/0chain_blobber.yaml | grep '^block_worker' | awk -F ' ' '{print $2}') + echo "> zcn is updated to: $zcn" +} + +install_debuggger() { + [ -d ../.vscode ] || mkdir -p ../.vscode + sed "s/Hostname/$hostname/g" launch.json > ../.vscode/launch.json + echo "debugbbers are installed" +} + +cleanAll() { + + cd $root + rm -rf ./data && echo "data is removed" +} + + +echo " " +echo "Please select which blobber/validator you will work on: " + +select i in "1" "2" "3" "clean all" "install debugers on .vscode/launch.json" "set hostname" "change zcn"; do + case $i in + "1" ) break;; + "2" ) break;; + "3" ) break;; + "clean all" ) cleanAll ;; + "install debugers on .vscode/launch.json" ) install_debuggger;; + "set hostname" ) set_hostname;; + "change zcn" ) change_zcn;; + esac +done + + +install_postgres () { + + echo Installing blobber_postgres in docker... + + [ ! "$(docker ps -a | grep blobber_postgres)" ] && docker run --name blobber_postgres --restart always -p 5432:5432 -e POSTGRES_PASSWORD=postgres -d postgres:11 + + [ -d "./data/blobber" ] || mkdir -p "./data/blobber" + + echo Initializing database + + [ -d "./data/blobber/sql" ] && rm -rf [ -d "./data/blobber/sql" ] + + cp -r ../sql "./data/blobber/" + cd "./data/blobber/sql" + + find . -name "*.sql" -exec sed -i '' "s/blobber_user/blobber_user/g" {} \; + find . -name "*.sql" -exec sed -i '' "s/blobber_meta/blobber_meta/g" {} \; + + + cd $root + [ -d "./data/blobber/bin" ] && rm -rf [ -d "./data/blobber/bin" ] + cp -r ../bin "./data/blobber/" + + + cd $root + + [ ! "$(docker ps -a | grep blobber_postgres_init)" ] && docker rm blobber_postgres_init --force + + + docker run --name blobber_postgres_init \ + --link blobber_postgres:postgres \ + -e POSTGRES_PORT=5432 \ + -e POSTGRES_HOST=postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -v $root/data/blobber/bin:/blobber/bin \ + -v $root/data/blobber/sql:/blobber/sql \ + postgres:11 bash /blobber/bin/postgres-entrypoint.sh + + docker rm blobber_postgres_init --force + +} + +prepareRuntime() { + + cd $root + [ -d ./data/blobber$i/config ] && rm -rf $root/data/blobber$i/config + cp -r ../config "./data/blobber$i/" + + cd ./data/blobber$i/config/ + + find . -name "*.yaml" -exec sed -i '' "s/blobber_user/blobber_user$i/g" {} \; + find . -name "*.yaml" -exec sed -i '' "s/blobber_meta/blobber_meta$i/g" {} \; + find . -name "*.yaml" -exec sed -i '' "s/postgres/127.0.0.1/g" {} \; + cd $root/data/blobber$i/ + + [ -d files ] || mkdir files + [ -d data ] || mkdir data + [ -d log ] || mkdir log +} + +start_blobber () { + + echo ">>>>>>>>>>>>>> Blobber $i <<<<<<<<<<<<<<<<" + + echo "[1/3] build blobber..." + cd ../code/go/0chain.net/blobber + go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/blobber . + + echo "[2/3] setup runtime..." + prepareRuntime; + cd $root + port="505$i" + grpc_port="3150$i" + + keys_file="../docker.local/keys_config/b0bnode${i}_keys.txt" + minio_file="../docker.local/keys_config/minio_config.txt" + config_dir="./data/blobber$i/config" + files_dir="./data/blobber$i/files" + log_dir="./data/blobber$i/log" + db_dir="./data/blobber$i/data" + + echo "[3/3] run blobber..." + + + ./data/blobber$i/blobber --port $port --grpc_port $grpc_port --hostname $hostname --deployment_mode 0 --keys_file $keys_file --files_dir $files_dir --log_dir $log_dir --db_dir $db_dir --minio_file $minio_file --config_dir $config_dir +} + +start_validator () { + + echo ">>>>>>>>>>>>>> Validator $i <<<<<<<<<<<<<<<<" + + echo "[1/3] build validator..." + cd ../code/go/0chain.net/validator + go build -v -tags "bn256 development" -gcflags="-N -l" -ldflags "-X 0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/validator . + + echo "[2/3] setup runtime" + prepareRuntime; + + cd $root + port="506$i" + hostname="localhost" + keys_file="../docker.local/keys_config/b0bnode${i}_keys.txt" + config_dir="./data/blobber$i/config" + log_dir="./data/blobber$i/log" + + echo "[3/3] run validator..." + ./data/blobber$i/validator --port $port -hostname $hostname --deployment_mode 0 --keys_file $keys_file --log_dir $log_dir --config_dir $config_dir +} + +clean () { + echo "Building blobber $i" + + cd $root + + rm -rf "./data/blobber$i" +} + + +echo " +********************************************** + Blobber/Validator $i +**********************************************" + +echo " " +echo "Please select what you will do: " + +select f in "install postgres" "start blobber" "start validator" "clean"; do + case $f in + "install postgres" ) install_postgres; break;; + "start blobber" ) start_blobber; break;; + "start validator" ) start_validator; break;; + "clean" ) clean; break;; + esac +done + + diff --git a/dev.local/launch.json b/dev.local/launch.json new file mode 100644 index 000000000..364b437d3 --- /dev/null +++ b/dev.local/launch.json @@ -0,0 +1,107 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "debug:blobber1", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/code/go/0chain.net/blobber", + "args": ["--port","5051", + "--grpc_port", "31501", + "--hostname","Hostname", + "--deployment_mode", "0" , + "--keys_file", "../../../../docker.local/keys_config/b0bnode1_keys.txt", + "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt", + + "--files_dir","../../../../dev.local/data/blobber1/files", + "--log_dir" ,"../../../../dev.local/data/blobber1/log", + "--db_dir", "../../../../dev.local/data/blobber1/data", + "--config_dir", "../../../../dev.local/data/blobber1/config" + ] + }, + { + "name": "debug:blobber2", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/code/go/0chain.net/blobber", + "args": ["--port","5052", + "--grpc_port", "31502", + "--hostname","Hostname", + "--deployment_mode", "0" , + "--keys_file", "../../../../docker.local/keys_config/b0bnode2_keys.txt", + "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt", + + "--files_dir","../../../../dev.local/data/blobber2/files", + "--log_dir" ,"../../../../dev.local/data/blobber2/log", + "--db_dir", "../../../../dev.local/data/blobber2/data", + "--config_dir", "../../../../dev.local/data/blobber2/config" + ] + }, + { + "name": "debug:blobber3", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/code/go/0chain.net/blobber", + "args": ["--port","5053", + "--grpc_port", "31503", + "--hostname","Hostname", + "--deployment_mode", "0" , + "--keys_file", "../../../../docker.local/keys_config/b0bnode3_keys.txt", + "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt", + + "--files_dir","../../../../dev.local/data/blobber3/files", + "--log_dir" ,"../../../../dev.local/data/blobber3/log", + "--db_dir", "../../../../dev.local/data/blobber3/data", + "--config_dir", "../../../../dev.local/data/blobber3/config" + ] + }, + { + "name": "debug:validator1", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/code/go/0chain.net/validator", + "args": ["--port","5061", + "--hostname","Hostname", + "--deployment_mode", "0" , + "--keys_file", "../../../../docker.local/keys_config/b0bnode1_keys.txt", + "--log_dir" ,"../../../../dev.local/data/blobber1/log", + "--config_dir", "../../../../dev.local/data/blobber1/config" + ] + }, + { + "name": "debug:validator2", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/code/go/0chain.net/validator", + "args": ["--port","5062", + "--hostname","Hostname", + "--deployment_mode", "0" , + "--keys_file", "../../../../docker.local/keys_config/b0bnode2_keys.txt", + "--log_dir" ,"../../../../dev.local/data/blobber2/log", + "--config_dir", "../../../../dev.local/data/blobber2/config" + ] + }, + { + "name": "debug:validator3", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/code/go/0chain.net/validator", + "args": ["--port","5063", + "--hostname","Hostname", + "--deployment_mode", "0" , + "--keys_file", "../../../../docker.local/keys_config/b0bnode3_keys.txt", + "--log_dir" ,"../../../../dev.local/data/blobber3/log", + "--config_dir", "../../../../dev.local/data/blobber3/config" + ] + } + ] +} \ No newline at end of file diff --git a/docker.aws/build.blobber/Dockerfile b/docker.aws/build.blobber/Dockerfile index 59f46404f..2c61ea11e 100644 --- a/docker.aws/build.blobber/Dockerfile +++ b/docker.aws/build.blobber/Dockerfile @@ -1,5 +1,5 @@ ARG image_tag -FROM golang:1.14.9-alpine3.12 as blobber_build +FROM golang:1.17.1-alpine3.14 as blobber_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -35,7 +35,7 @@ ARG go_bls_tag RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X 0chain.net/core/build.BuildTag=${image_tag}" # Copy the build artifact into a minimal runtime image: -FROM golang:1.11.4-alpine3.8 +FROM golang:1.17.1-alpine3.14 RUN apk add gmp gmp-dev openssl-dev COPY --from=blobber_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/docker.aws/build.validator/Dockerfile b/docker.aws/build.validator/Dockerfile index 4c2c0a15e..84c964349 100644 --- a/docker.aws/build.validator/Dockerfile +++ b/docker.aws/build.validator/Dockerfile @@ -1,5 +1,5 @@ ARG image_tag -FROM golang:1.14.9-alpine3.12 as validator_build +FROM golang:1.17.1-alpine3.14 as validator_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -35,7 +35,7 @@ ARG go_bls_tag RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X 0chain.net/core/build.BuildTag=${image_tag}" # Copy the build artifact into a minimal runtime image: -FROM golang:1.11.4-alpine3.8 +FROM golang:1.17.1-alpine3.14 RUN apk add gmp gmp-dev openssl-dev COPY --from=validator_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/docker.local/Dockerfile b/docker.local/Dockerfile index ef7b728b0..193fff650 100644 --- a/docker.local/Dockerfile +++ b/docker.local/Dockerfile @@ -1,4 +1,6 @@ -FROM golang:1.14.9-alpine3.12 as blobber_build +FROM golang:1.17.1-alpine3.14 as blobber_build + +LABEL zchain="blobber" RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -37,7 +39,7 @@ ENV GIT_COMMIT=$GIT_COMMIT RUN go build -v -tags "bn256 development" -gcflags "all=-N -l" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.14.9-alpine3.12 +FROM golang:1.17.1-alpine3.14 RUN apk add gmp gmp-dev openssl-dev git COPY --from=blobber_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/docker.local/Dockerfile.dev b/docker.local/Dockerfile.dev index a2c38d790..83ca7019a 100644 --- a/docker.local/Dockerfile.dev +++ b/docker.local/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.14.9-alpine3.12 as blobber_build +FROM golang:1.17.1-alpine3.14 as blobber_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -38,7 +38,7 @@ ENV GIT_COMMIT=$GIT_COMMIT RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.14.9-alpine3.12 +FROM golang:1.17.1-alpine3.14 RUN apk add gmp gmp-dev openssl-dev COPY --from=blobber_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/docker.local/IntegrationTestsBlobberDockerfile b/docker.local/IntegrationTestsBlobberDockerfile index 77c37028f..e7d0c1bb2 100644 --- a/docker.local/IntegrationTestsBlobberDockerfile +++ b/docker.local/IntegrationTestsBlobberDockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.9-alpine3.12 as blobber_build +FROM golang:1.17.1-alpine3.14 as blobber_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -31,7 +31,7 @@ WORKDIR $SRC_DIR/go/0chain.net/blobber RUN go build -v -tags "bn256 development integration_tests" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.11.4-alpine3.8 +FROM golang:1.17.1-alpine3.14 RUN apk add gmp gmp-dev openssl-dev COPY --from=blobber_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/docker.local/ValidatorDockerfile b/docker.local/ValidatorDockerfile index cd5f594a1..f451e522f 100644 --- a/docker.local/ValidatorDockerfile +++ b/docker.local/ValidatorDockerfile @@ -1,4 +1,6 @@ -FROM golang:1.14.9-alpine3.12 as validator_build +FROM golang:1.17.1-alpine3.14 as validator_build + +LABEL zchain="validator" RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -35,7 +37,7 @@ WORKDIR $SRC_DIR/code/go/0chain.net/validator RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.11.4-alpine3.8 +FROM golang:1.17.1-alpine3.14 RUN apk add gmp gmp-dev openssl-dev COPY --from=validator_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/docker.local/ValidatorDockerfile.dev b/docker.local/ValidatorDockerfile.dev index 1841a035f..a574c9841 100644 --- a/docker.local/ValidatorDockerfile.dev +++ b/docker.local/ValidatorDockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.14.9-alpine3.12 as validator_build +FROM golang:1.17.1-alpine3.14 as validator_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -36,7 +36,7 @@ WORKDIR $SRC_DIR/go/0chain.net/validator RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.11.4-alpine3.8 +FROM golang:1.17.1-alpine3.14 RUN apk add gmp gmp-dev openssl-dev COPY --from=validator_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile new file mode 100644 index 000000000..f39a5860c --- /dev/null +++ b/docker.local/base.Dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.17.1-alpine3.14 as blobber_base + +LABEL zchain="blobber" + +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories + +RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep + +# Install Herumi's cryptography +RUN apk add gmp gmp-dev openssl-dev && \ + cd /tmp && \ + wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ + mv mcl* mcl && \ + make -C mcl -j $(nproc) lib/libmclbn256.so install && \ + cp mcl/lib/libmclbn256.so /usr/local/lib && \ + rm -R /tmp/mcl + +RUN git clone https://github.com/herumi/bls /tmp/bls && \ + cd /tmp/bls && \ + git submodule init && \ + git submodule update && \ + make -j $(nproc) install && \ + cd - && \ + rm -R /tmp/bls diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh new file mode 100755 index 000000000..252909943 --- /dev/null +++ b/docker.local/bin/build.base.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +GIT_COMMIT=$(git rev-list -1 HEAD) +echo $GIT_COMMIT + + +docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base \ No newline at end of file diff --git a/docker.local/bin/build.blobber.sh b/docker.local/bin/build.blobber.sh index 4cd4fda82..64daed5d1 100755 --- a/docker.local/bin/build.blobber.sh +++ b/docker.local/bin/build.blobber.sh @@ -17,12 +17,9 @@ do esac done -docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/ValidatorDockerfile . -t validator -docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/Dockerfile . -t blobber +# [ -d ./gosdk ] && rm -rf gosdk +# cp -r ../gosdk ./ -for i in $(seq 1 6); -do - BLOBBER=$i docker-compose -p blobber$i -f docker.local/docker-compose.yml build --force-rm -done -docker.local/bin/sync_clock.sh +docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/blobber.Dockerfile . -t blobber + diff --git a/docker.local/bin/build.validator.sh b/docker.local/bin/build.validator.sh new file mode 100755 index 000000000..162828098 --- /dev/null +++ b/docker.local/bin/build.validator.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +GIT_COMMIT=$(git rev-list -1 HEAD) +echo $GIT_COMMIT + +cmd="build" + +for arg in "$@" +do + case $arg in + -m1|--m1|m1) + echo "The build will be performed for Apple M1 chip" + cmd="buildx build --platform linux/amd64" + shift + ;; + esac +done + +# [ -d ./gosdk ] && rm -rf gosdk +# cp -r ../gosdk ./ + +docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/validator.Dockerfile . -t validator + diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile new file mode 100644 index 000000000..528599e4a --- /dev/null +++ b/docker.local/blobber.Dockerfile @@ -0,0 +1,35 @@ +FROM blobber_base as blobber_build + +LABEL zchain="blobber" + + +ENV SRC_DIR=/0chain +ENV GO111MODULE=on +#ENV GOPROXY=https://goproxy.cn,direct + +# Download the dependencies: +# Will be cached if we don't change mod/sum files +COPY . $SRC_DIR +# COPY ./gosdk /gosdk + +RUN cd $SRC_DIR/ && go mod download + + +WORKDIR $SRC_DIR/code/go/0chain.net/blobber + +ARG GIT_COMMIT +ENV GIT_COMMIT=$GIT_COMMIT +RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" + +# Copy the build artifact into a minimal runtime image: +FROM golang:1.17.1-alpine3.14 +RUN apk add gmp gmp-dev openssl-dev git +COPY --from=blobber_build /usr/local/lib/libmcl*.so \ + /usr/local/lib/libbls*.so \ + /usr/local/lib/ + + +ENV APP_DIR=/blobber +WORKDIR $APP_DIR +COPY --from=blobber_build /0chain/code/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber + diff --git a/docker.local/docker-compose.yml b/docker.local/docker-compose.yml index be11ce879..baa238b28 100644 --- a/docker.local/docker-compose.yml +++ b/docker.local/docker-compose.yml @@ -8,6 +8,8 @@ services: default: ports: - "543${BLOBBER}:5432" + labels: + zchain: "postgres" postgres-post: image: postgres:11 environment: @@ -17,6 +19,8 @@ services: volumes: - ../bin:/blobber/bin - ../sql:/blobber/sql + labels: + zchain: "postgres-post" command: bash /blobber/bin/postgres-entrypoint.sh links: - postgres:postgres @@ -35,6 +39,8 @@ services: - ./keys_config:/blobber/keysconfig ports: - "506${BLOBBER}:506${BLOBBER}" + labels: + zchain: "validator" command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --log_dir /blobber/log networks: default: @@ -56,6 +62,8 @@ services: ports: - "505${BLOBBER}:505${BLOBBER}" - "703${BLOBBER}:703${BLOBBER}" + labels: + zchain: "blobber" command: ./bin/blobber --port 505${BLOBBER} --grpc_port 703${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data --minio_file keys_config/minio_config.txt networks: default: diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile new file mode 100644 index 000000000..58cbb9c4a --- /dev/null +++ b/docker.local/validator.Dockerfile @@ -0,0 +1,30 @@ +FROM blobber_base as validator_build + +LABEL zchain="validator" + + +ENV SRC_DIR=/blobber +ENV GO111MODULE=on +#ENV GOPROXY=https://goproxy.cn,direct + + +# Download the dependencies: +# Will be cached if we don't change mod/sum files +COPY . $SRC_DIR +# COPY ./gosdk /gosdk + +RUN cd $SRC_DIR/ && go mod download + +WORKDIR $SRC_DIR/code/go/0chain.net/validator + +RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" + +# Copy the build artifact into a minimal runtime image: +FROM golang:1.17.1-alpine3.14 +RUN apk add gmp gmp-dev openssl-dev +COPY --from=validator_build /usr/local/lib/libmcl*.so \ + /usr/local/lib/libbls*.so \ + /usr/local/lib/ +ENV APP_DIR=/blobber +WORKDIR $APP_DIR +COPY --from=validator_build $APP_DIR/code/go/0chain.net/validator/validator $APP_DIR/bin/validator \ No newline at end of file diff --git a/docs/src/stream_hash.puml b/docs/src/stream_hash.puml new file mode 100644 index 000000000..4a193df5a --- /dev/null +++ b/docs/src/stream_hash.puml @@ -0,0 +1,157 @@ +@startuml Stream Hash (Current) + +actor Client + +title Stream Hash (Current) +Client -> Blockchain : Make allocation request (erasure code, size, expiry) +Blockchain -> Blockchain : Assigns the blobbers \ncreates the read/write pools for allocation +Blockchain --> Client : Allocation status, if success, list of blobbers selected + + +loop till end +group upload chunk [ read bytes with (DataShards * chunk size)/renaming size ] + Client -> "FileHasher:SHA1" : Write bytes - original bytes has to be saved in memory (unencoded, unencrypted) + note right + 1. it prevents stream upload + 2. bytes copy +1 + end note + group sharding [ 1.. DataShards+ParityShards ] + Client -> "ChunkHasher:SHA1" : Write bytes - bytes can be released once it is uploaded (encoded, encrypted) + Client -> "ShardHasher:SHA1" : Write bytes - bytes has to be saved in memory (encoded, encrypted) + note right + 1. it prevents stream upload + 2. bytes copy +1 + end note + + group outsourcing attack protection [ 1..1024 ] + Client -> "SivaHasher:Merkle+SHA3" : Write bytes to leaf node - MerkleLeaves[i/64].Write(bytes[i:i+64]) + note right + 1. it prevents stream upload. Because leaf is hashed with sha3.New256 + 2. bytes copy +1 + end note + end + + alt first chunk + "ChunkHasher:SHA1" -> Client: compute chunk bytes hash + Client -> Blobber : PATCH request - Upload the shard bytes (path, metadata, connection id) and thumbnail shard bytes if it has + + else final chunk + "ChunkHasher:SHA1" -> Client: compute chunk bytes hash + "FileHasher:SHA1" -> Client : compute sha1 hash for full original file content + note right + Memory can be releaed now + end note + "ShardHasher:SHA1" -> Client : compute sha1 hash for all shard bytes that has been uploaded to this blobber + note right + Memory can be releaed now + end note + "SivaHasher:Merkle+SHA3" -> Client : compute merkel root wiht MarkelLeaves + note right + Memory can be releaed now + end note + Client -> Blobber : PATCH request - Upload the shard bytes (path, metadata, connection id) and final flag + else streaming chunk + "ChunkHasher:SHA1" -> Client: compute chunk bytes hash + Client -> Blobber : PATCH request - Upload the shard bytes (path, metadata, connection id) + end + end + Blobber --> Client : Upload successful +end + +end +Client -> Blobber : commit the connection and pass the writemarker (allocation root, prev allocation root, upload size) +Blobber -> Blobber : validates the write marker, commits any delete tokens +Blobber --> Client : commit success / failure + + + +Blobber -> Blockchain : redeem the write marker in the order received +note right +async redeemption +not blocking uploads +end note + +Blockchain -> Blockchain : Move tokens from write pool to challenge pool + +@enduml + + +@startuml Stream Hash (New) + +actor Client + +title Stream Hash (New) +Client -> Blockchain : Make allocation request (erasure code, size, expiry) +Blockchain -> Blockchain : Assigns the blobbers \ncreates the read/write pools for allocation +Blockchain --> Client : Allocation status, if success, list of blobbers selected + + +loop till end +group upload chunk [ read bytes with (DataShards * chunk size)/renaming size ] + Client -> "FileHasher:StreamMerkle+SHA1" : Compute chunk bytes instantly with sha1 as merkel leaf - bytes can be released after hashing (unencoded, unencrypted) + note right + Merkle tree is auto balanced as much as possible when a new leaf is pushed.Memory is took as less as possible. The tree is stateful, is easy to save and reload. + end note + group sharding [ 1.. DataShards+ParityShards ] + + group outsourcing attack protection [ 1..1024 ] + Client -> "ShardHasher:SivaMerkle+(StreamMerkle+SHA1)" : Compute 1/1024 bytes instanty, and append to leaf's StreamMerkleHasher as leaf's leaf node + note right + SivaMerkle+(StreamMerkle+SHA1) is a merkle tree with 1024 leaves. A leaf is a StreamMerkleHasher tree with many leaves that is hashed with sha1 + end note + end + + alt first chunk + "ChunkHasher:SHA1" -> Client: compute chunk bytes hash + Client -> Blobber : PATCH request - Upload the shard bytes (path, metadata, connection id) and thumbnail shard bytes if it has + else final chunk + "ChunkHasher:SHA1" -> Client: compute chunk bytes hash + "FileHasher:StreamMerkle+SHA1" -> Client : compute merkel root with states + note right + States can be releaed now + end note + + "ShardHasher:SivaMerkle+(StreamMerkle+SHA1)" -> Client : compute 1024 leaf's merkel root first, and compute top merkel root based 1024 merkel roots + note right + States can be releaed now + end note + Client -> Blobber : PATCH request - Upload the shard bytes (path, metadata, connection id) and final flag + else streaming chunk + "ChunkHasher:SHA1" -> Client: compute chunk bytes hash + Client -> Blobber : PATCH request - Upload the shard bytes (path, metadata, connection id) + end + end + Blobber --> Client : Upload successful +end + +end +Client -> Blobber : commit the connection and pass the writemarker (allocation root, prev allocation root, upload size) +Blobber -> Blobber : validates the write marker, commits any delete tokens +Blobber --> Client : commit success / failure + + + +Blobber -> Blockchain : redeem the write marker in the order received +note right +async redeemption +not blocking uploads +end note + +Blockchain -> Blockchain : Move tokens from write pool to challenge pool + +@enduml + +@startuml StandardMerkleHahser + +title StandardMerkleHahser + + +@enduml + + +@startuml StreamMerkleHahser + +title StreamMerkleHahser + + +@enduml \ No newline at end of file diff --git a/docs/src/upload_flow.puml b/docs/src/upload_flow.puml new file mode 100644 index 000000000..f22821f2b --- /dev/null +++ b/docs/src/upload_flow.puml @@ -0,0 +1,36 @@ +@startuml Upload / Delete Flow + +actor Client + +title Upload/Delete flow +Client -> Blockchain : Make allocation request (erasure code, size, expiry) +Blockchain -> Blockchain : Assigns the blobbers \ncreates the read/write pools for allocation +Blockchain --> Client : Allocation status, if success, list of blobbers selected + + +loop till done +group upload + Client -> Blobber : POST request - Upload the file (path, metadata, connection id) + Blobber --> Client : Upload successful +end +group delete + Client -> Blobber : DELETE request - Delete the file and pass delete token (path hash, file hash from list api) + Blobber -> Blobber : Validate the delete token and stores with New state + Blobber --> Client : delete successful +end +end +Client -> Blobber : commit the connection and pass the writemarker (allocation root, prev allocation root, upload size) +Blobber -> Blobber : validates the write marker, commits any delete tokens +Blobber --> Client : commit success / failure + + + +Blobber -> Blockchain : redeem the write marker in the order received +note right +async redeemption +not blocking uploads +end note + +Blockchain -> Blockchain : Move tokens from write pool to challenge pool + +@enduml \ No newline at end of file diff --git a/go.mod b/go.mod index 1744366f0..751f34318 100644 --- a/go.mod +++ b/go.mod @@ -1,30 +1,31 @@ module github.com/0chain/blobber +go 1.16 + require ( - github.com/0chain/errors v1.0.2 - github.com/0chain/gosdk v1.2.85-0.20210905084101-4d9d75eb73f2 + github.com/0chain/errors v1.0.3 + github.com/0chain/gosdk v1.2.88 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/didip/tollbooth v4.0.2+incompatible github.com/go-ini/ini v1.55.0 // indirect - github.com/gorilla/handlers v1.4.2 - github.com/gorilla/mux v1.7.3 + github.com/gorilla/handlers v1.5.1 + github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0 - github.com/herumi/bls-go-binary v0.0.0-20191119080710-898950e1a520 + github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.14.0 github.com/jackc/pgproto3/v2 v2.0.4 // indirect github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/mapstructure v1.3.1 - github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce github.com/rs/cors v1.8.0 // indirect github.com/selvatico/go-mocket v1.0.7 github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 - go.uber.org/zap v1.16.0 + go.uber.org/zap v1.19.0 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb google.golang.org/grpc v1.36.0 @@ -34,12 +35,9 @@ require ( gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d - gorm.io/driver/mysql v0.3.1 gorm.io/driver/postgres v1.0.0 gorm.io/gorm v1.20.4 nhooyr.io/websocket v1.8.7 // indirect ) -go 1.13 - -//replace github.com/0chain/gosdk => ../gosdk +// replace github.com/0chain/gosdk => ../gosdk diff --git a/go.sum b/go.sum index 26206f411..6e1914eef 100644 --- a/go.sum +++ b/go.sum @@ -36,10 +36,11 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/0chain/errors v1.0.2 h1:IIUMeh/qFlqDcyHesjU92CpRMVz9dIQWAtZooqrYinA= github.com/0chain/errors v1.0.2/go.mod h1:5t76jLb56TKfg/K2VD+eUMmNZJ42QsIRI8KzWuztwU4= -github.com/0chain/gosdk v1.2.85-0.20210905084101-4d9d75eb73f2 h1:R89Ttt4+Qro5prWnA4YyuHIW3QJRdgIYD90BEG5Iaq4= -github.com/0chain/gosdk v1.2.85-0.20210905084101-4d9d75eb73f2/go.mod h1:Bl/wsHdlktgXybdzkHPfaf6ATe7mPmew/xF3ki2gskQ= +github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= +github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= +github.com/0chain/gosdk v1.2.88 h1:rllzNvDvLP9iDXI6jYT9v3aBgeucGvb1uOpA9Af2AjQ= +github.com/0chain/gosdk v1.2.88/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -59,17 +60,15 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/sarama v1.28.0/go.mod h1:j/2xTrU39dlzBmsxF1eQ2/DdWrxyBCl6pzz7a81o/ZY= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -78,9 +77,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= @@ -90,17 +87,12 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= -github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/aristanetworks/goarista v0.0.0-20210308203447-b196d8410f1d/go.mod h1:drswc1gdKErwWsW+gV2R5ELcuHehg5pZD2tat4B65Ik= -github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= @@ -112,6 +104,8 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7 github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -120,7 +114,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= @@ -140,9 +133,7 @@ github.com/bufbuild/buf v0.37.0/go.mod h1:lQ1m2HkIaGOFba6w/aC3KYBHhKEOESP3gaAEpS github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.0.0/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -153,22 +144,19 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/consensys/bavard v0.1.8-0.20210105233146-c16790d2aa8b/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= -github.com/consensys/goff v0.3.10/go.mod h1:xTldOBEHmFiYS0gPXd3NsaEqZWlnmeWcRLWgD3ba3xc= -github.com/consensys/gurvy v0.3.8/go.mod h1:sN75xnsiD593XnhbhvG2PkOy194pZBzqShWF/kwuW/g= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -176,10 +164,12 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -188,12 +178,17 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= +github.com/dgraph-io/badger/v3 v3.2103.1/go.mod h1:dULbq6ehJ5K0cGW/1TQ9iSfUk0gbSiToDWmWmTsJ53E= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M= github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY= @@ -201,43 +196,36 @@ github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55k github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/gosigar v0.14.1/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= -github.com/ethereum/go-ethereum v1.10.1/go.mod h1:E5e/zvdfUVr91JZ0AwjyuJM3x+no51zZJRz61orLLSk= -github.com/ethereum/go-ethereum v1.10.3 h1:SEYOYARvbWnoDl1hOSks3ZJQpRiiRJe8ubaQGJQwq0s= -github.com/ethereum/go-ethereum v1.10.3/go.mod h1:99onQmSd1GRGOziyGldI41YQb7EESX3Q4H41IfJgIQQ= -github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/ethereum/go-ethereum v1.10.8 h1:0UP5WUR8hh46ffbjJV7PK499+uGEyasRIfffS0vy06o= +github.com/ethereum/go-ethereum v1.10.8/go.mod h1:pJNuIUYfX5+JKzSD/BTdNsvJSZ1TJqmz0dVyXMAbf6M= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -247,6 +235,7 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -254,15 +243,14 @@ github.com/go-ini/ini v1.55.0 h1:0wVcG9udk2C3TGgmdIGKK9ScOZHZB5nbG+gwji9fhhc= github.com/go-ini/ini v1.55.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -333,13 +321,13 @@ github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -362,7 +350,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/protobuf v3.14.0+incompatible/go.mod h1:lUQ9D1ePzbH2PrIS7ob/bjm9HXyH5WHB0Akwh7URreM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -375,19 +362,16 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -407,20 +391,20 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -429,30 +413,29 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/herumi/bls-go-binary v0.0.0-20191119080710-898950e1a520 h1:3ek8BJos3JW72rvPzGAWZwJ/iXjOyPSCUI4nAFnTPvg= -github.com/herumi/bls-go-binary v0.0.0-20191119080710-898950e1a520/go.mod h1:uTBfU/n3h1aOYIl5nNTbLn5dUfNkF1P97JTaz3bdvro= +github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 h1:TV+x87PxW2BcEzpX6yWK0mBEGc9NFppsihFszD6QmyA= +github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= -github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= -github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88 h1:bcAj8KroPf552TScjFPIakjH2/tdIrIH8F+cc4v4SRo= -github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/improbable-eng/grpc-web v0.14.0 h1:GdoK+cXABdB+1keuqsV1drSFO2XLYIxqt/4Rj8SWGBk= github.com/improbable-eng/grpc-web v0.14.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= @@ -514,12 +497,6 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= -github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= -github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -532,27 +509,22 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kilic/bls12-381 v0.0.0-20201226121925-69dacb279461/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -561,11 +533,10 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.9 h1:5OCMOdde1TCT2sookEuVeEZzA8bmRSFV3AwPDZAG8AA= -github.com/klauspost/compress v1.11.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -578,7 +549,6 @@ github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 h1:R7RAW1p8wjhlHKFhS4 github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20/go.mod h1:sh5SGGmQVGUkWDnxevz0I2FJ4TeC18hRPRjKVBMb2kA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -589,7 +559,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leanovate/gopter v0.2.8/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= @@ -604,19 +575,24 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magma/augmented-networks/accounting/protos v0.1.1/go.mod h1:Hpfg8aAxldUN7qlVtR5xwlAf8pcetFm8DWwRKZsh2J4= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -625,7 +601,6 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -634,8 +609,6 @@ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miguelmota/go-ethereum-hdwallet v0.0.1 h1:DWqgZtKWTGcHR5QsprMJItZiJ2xVEQTv640r597ul8M= -github.com/miguelmota/go-ethereum-hdwallet v0.0.1/go.mod h1:iowKavXnc0NVNiv/UKYYBo3SjADph5PUvYQTjOIV9as= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -656,9 +629,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -669,33 +641,22 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/openconfig/gnmi v0.0.0-20210226144353-8eae1937bf84/go.mod h1:H/20NXlnWbCPFC593nxpiKJ+OU//7mW7s7Qk7uVdg3Q= -github.com/openconfig/goyang v0.0.0-20200115183954-d0a48929f0ea/go.mod h1:dhXaV0JgHJzdrHi2l+w0fZrwArtXL7jEFoiqLEdmkvU= -github.com/openconfig/reference v0.0.0-20201210185750-72ca4cfd4abd/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= -github.com/openconfig/ygot v0.6.0/go.mod h1:o30svNf7O0xK+R35tlx95odkDmZWS9JyWWQSmIhqwAs= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -710,18 +671,15 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -737,36 +695,25 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= @@ -774,15 +721,15 @@ github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeC github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= -github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -793,7 +740,6 @@ github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfP github.com/selvatico/go-mocket v1.0.7 h1:jbVa7RkoOCzBanQYiYF+VWgySHZogg25fOIKkM38q5k= github.com/selvatico/go-mocket v1.0.7/go.mod h1:7bSWzuNieCdUlanCVu3w0ppS0LvDtPAZmKBIlhoTcp8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -803,7 +749,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -811,25 +756,25 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20201006035406-b97b5ead31f7/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -847,13 +792,9 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= -github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tjfoc/gmsm v1.4.0/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= @@ -866,19 +807,19 @@ github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2n github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xtaci/kcp-go v5.4.20+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= -github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -901,6 +842,7 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -908,6 +850,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -920,11 +864,13 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -937,14 +883,11 @@ golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -953,7 +896,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -977,22 +919,18 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1026,14 +964,13 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1049,10 +986,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1061,6 +996,7 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1083,7 +1019,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1099,23 +1034,19 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 h1:EjgCl+fVlIaPJSori0ikSz3uV0DOHKWOJFpv1sAAhBM= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1124,8 +1055,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1158,6 +1090,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1168,7 +1101,6 @@ golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1263,7 +1195,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb h1:hcskBH5qZCOa7WpTUFUFvoebnSFZBYpjykLtjIp9DVk= @@ -1286,7 +1217,6 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0-dev.0.20201218190559-666aea1fb34c/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= @@ -1309,7 +1239,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1325,11 +1254,6 @@ gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:a gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.61.0 h1:LBCdW4FmFYL4s/vDZD1RQYX7oAR6IjujCYgMdbHBR10= gopkg.in/ini.v1 v1.61.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -1337,9 +1261,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= @@ -1349,7 +1271,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -1383,7 +1304,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= diff --git a/sql/15-add-chunk_size-to-reference_objects.sql b/sql/15-add-chunk_size-to-reference_objects.sql new file mode 100644 index 000000000..a2530e3f0 --- /dev/null +++ b/sql/15-add-chunk_size-to-reference_objects.sql @@ -0,0 +1,9 @@ +-- +-- Add chunk_size column to reference_objects table. +-- + +-- pew-pew +\connect blobber_meta; + + +ALTER TABLE reference_objects ADD COLUMN chunk_size INT NOT NULL DEFAULT 65536; From 6672504cd7730ee9ed3df52977aec0ef16925707 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 23 Sep 2021 10:20:56 +0800 Subject: [PATCH 004/161] added codeql-analysis (#347) --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..55a5c7a7e --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ staging, master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ staging ] + schedule: + - cron: '43 17 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 2e6e118c5dfbd45cc88b616d09ff59712c4cd558 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 23 Sep 2021 19:07:23 +0800 Subject: [PATCH 005/161] Fix/clean GitHub action (#348) * fix(devops):clean Dockerize, and enabled unit tests * fix(devops):clean github actions * fix(devops): replace runner with self-hosted runners * fix(devops): fixed network issue * fix(devops): removed network create script * fix(devops): run integration-test on ubuntu-20.04 * fix(devops): run CodeQL on self-hosted runners * fix(devops): run CodeQL on [self-hosted, build] * fix(devops): try run CodeQL/integration-tests on [self-hosted, build] * fix(devops): run CodeQL/integration-tests on github runner ubuntu-20.04 * fix(devops): use external network in integration-tesets * fix(devops): reverted testnet0 to external --- .github/workflows/benchmark.yml | 11 +++---- .github/workflows/build.yml | 4 +-- .github/workflows/ci.yml | 43 +++---------------------- .github/workflows/cicd.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dev-1.yml | 2 +- .github/workflows/dockerize.yml | 25 ++++++-------- .github/workflows/tests.yml | 39 +++++++++++----------- Makefile | 4 +-- docker.local/b0docker-compose-debug.yml | 5 +++ docker.local/b0docker-compose.yml | 6 ++++ docker.local/docker-compose.yml | 6 ++++ docker.local/p0docker-compose.yml | 5 +++ 13 files changed, 67 insertions(+), 87 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index daddae876..b5616a4fd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,20 +2,21 @@ name: Benchmark tests on: push: - # branches: [ master ] + branches: [ master, staging, stream ] pull_request: - # branches: [ master ] + branches: [ master, staging, stream ] jobs: benchmark: - runs-on: ubuntu-latest + runs-on: [self-hosted, load-test] steps: - - name: Setup go 1.16 uses: actions/setup-go@v2 with: go-version: '1.16' # The Go version to download (if necessary) and use. + - name: Clone blobber + uses: actions/checkout@v2 - name: Set GITHUB_ENV run: | @@ -23,8 +24,6 @@ jobs: echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - name: Clone blobber - uses: actions/checkout@v2 # - name: Clone gosdk # run: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 166e61899..ff57da5c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ env: jobs: dockerize_blobber: - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] steps: - uses: actions/checkout@v2 @@ -56,7 +56,7 @@ jobs: BRANCH: ${{ steps.get_version.outputs.BRANCH }} dockerize_validator: - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b03989169..fb4ecc7ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,24 +28,6 @@ jobs: - name: Clone Blobber uses: actions/checkout@v2 - # - uses: satackey/action-docker-layer-caching@v0.0.11 - # continue-on-error: true - - - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - # git clone gosdk with same branch - # - name: Clone gosdk - # run: | - # [ -d $GOSDK ] && rm -rf $GOSDK - # git clone https://github.com/0chain/gosdk.git $GOSDK - # cd $GOSDK - # git checkout $BRANCH - - name: Build test environment and run tests run: | sed -i '/#expose_ci_port/c\ ports:\n - "5432:5432"' ./docker.local/b0docker-compose.yml @@ -59,10 +41,10 @@ jobs: cd ../.. go16=$(which go) - sudo $go16 test ./... -args integration; + sudo $go16 test -tags bn256 ./... -args integration; golangci: name: lint - runs-on: ubuntu-latest + runs-on: [self-hosted, tests-suite] steps: - name: Setup go 1.16 uses: actions/setup-go@v2 @@ -70,21 +52,6 @@ jobs: go-version: '1.16' # The Go version to download (if necessary) and use. - uses: actions/checkout@v2 - - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - # git clone gosdk with same branch - # - name: Clone gosdk - # run: | - # [ -d $GOSDK ] && rm -rf $GOSDK - # git clone https://github.com/0chain/gosdk.git $GOSDK - # cd $GOSDK - # git checkout $BRANCH - - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: @@ -92,7 +59,7 @@ jobs: args: --timeout=2m0s buf-build: - runs-on: ubuntu-latest + runs-on: [self-hosted, tests-suite] steps: - name: Check out code uses: actions/checkout@master @@ -109,7 +76,7 @@ jobs: run: make local dockerize_blobber: - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] needs: integration_test if: github.event_name == 'release' && github.event.action == 'published' steps: @@ -133,7 +100,7 @@ jobs: TAG: ${{ steps.get_version.outputs.VERSION }} dockerize_validator: - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] needs: integration_test if: github.event_name == 'release' && github.event.action == 'published' steps: diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index af12b0992..2f1f64982 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -10,7 +10,7 @@ on: jobs: deploy: - runs-on: ubuntu-20.04 + runs-on: [self-hosted] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 55a5c7a7e..216973208 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 permissions: actions: read contents: read diff --git a/.github/workflows/dev-1.yml b/.github/workflows/dev-1.yml index 7a3392c57..62ef2fec3 100644 --- a/.github/workflows/dev-1.yml +++ b/.github/workflows/dev-1.yml @@ -10,7 +10,7 @@ on: jobs: deploy: - runs-on: ubuntu-20.04 + runs-on: [self-hosted] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml index b608d9749..0d5759db9 100644 --- a/.github/workflows/dockerize.yml +++ b/.github/workflows/dockerize.yml @@ -3,15 +3,10 @@ name: Dockerize on: push: branches: - - dayi/main - - dayi/qa - - dayi/dev - pull_request: - - dayi/main - - dayi/qa - - dayi/dev - # tags: - # - '*' + - staging + - stream + tags: + - '*' env: @@ -20,7 +15,7 @@ env: jobs: build_base: - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] steps: - name: Setup go 1.16 uses: actions/setup-go@v2 @@ -58,7 +53,7 @@ jobs: build_blobber: needs: build_base - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] steps: - name: Set GITHUB_ENV run: | @@ -85,8 +80,6 @@ jobs: - name: Load blobber_base run: | docker load --input /tmp/0chain/blobber_base.tar - - #docker image ls -a - name: Build blobber run: ./docker.local/bin/build.blobber.sh @@ -104,7 +97,7 @@ jobs: publish_blobber: needs: build_blobber - runs-on: ubuntu-20.04 + runs-on: [self-hosted] steps: - name: Set GITHUB_ENV run: | @@ -140,7 +133,7 @@ jobs: build_validator: needs: build_base - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] steps: - name: Set GITHUB_ENV run: | @@ -186,7 +179,7 @@ jobs: publish_validator: needs: build_validator - runs-on: ubuntu-20.04 + runs-on: [self-hosted] steps: - name: Set GITHUB_ENV run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce181c9ef..39df09bb7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,33 +3,32 @@ name: Tests on: push: branches: - - dayi/main - - dayi/qa - - dayi/dev + - master + - staging + - stream pull_request: - - dayi/main - - dayi/qa - - dayi/dev - # tags: - # - '*' + - master + - staging + - stream jobs: unit_tests: - runs-on: ubuntu-20.04 + runs-on: [self-hosted, load-test] steps: - name: Setup go 1.16 uses: actions/setup-go@v2 with: go-version: '1.16' # The Go version to download (if necessary) and use. - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - + - name: Clone blobber uses: actions/checkout@v2 + # - name: Set GITHUB_ENV + # run: | + # echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + # echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + # echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + # - name: Clone gosdk # run: | # [ -d $GOSDK ] && rm -rf $GOSDK @@ -40,8 +39,8 @@ jobs: - name: Run tests run: | cd $GITHUB_WORKSPACE/code/go/0chain.net/ - go test -race -coverprofile=coverage.txt -covermode=atomic ./... - - name: Codecov - run: | - cd $GITHUB_WORKSPACE/code/go/0chain.net/ - bash <(curl -s https://codecov.io/bash) \ No newline at end of file + go test -tags bn256 -race -coverprofile=coverage.txt -covermode=atomic ./... + # - name: Codecov + # run: | + # cd $GITHUB_WORKSPACE/code/go/0chain.net/ + # bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/Makefile b/Makefile index aa62d38b0..6415b08e4 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ .PHONY: test test: - go test ./...; + go test -tags bn256 ./...; .PHONY: lint lint: @@ -22,7 +22,7 @@ lint: .PHONY: integration-tests integration-tests: - go test ./... -args integration; + go test -tags bn256 ./... -args integration; diff --git a/docker.local/b0docker-compose-debug.yml b/docker.local/b0docker-compose-debug.yml index d185d0b3d..8525d3500 100644 --- a/docker.local/b0docker-compose-debug.yml +++ b/docker.local/b0docker-compose-debug.yml @@ -76,6 +76,11 @@ networks: driver: bridge testnet0: external: true + # driver: bridge + # ipam: + # config: + # - subnet: 198.18.0.0/15 + # - gateway: 198.18.0.255 volumes: data: diff --git a/docker.local/b0docker-compose.yml b/docker.local/b0docker-compose.yml index d0c04bbcb..f3e392acc 100644 --- a/docker.local/b0docker-compose.yml +++ b/docker.local/b0docker-compose.yml @@ -74,6 +74,12 @@ networks: driver: bridge testnet0: external: true + # driver: bridge + # ipam: + # config: + # - subnet: 198.18.0.0/15 + # - gateway: 198.18.0.255 + volumes: data: diff --git a/docker.local/docker-compose.yml b/docker.local/docker-compose.yml index baa238b28..5d250bf16 100644 --- a/docker.local/docker-compose.yml +++ b/docker.local/docker-compose.yml @@ -75,6 +75,12 @@ networks: driver: bridge testnet0: external: true + # driver: bridge + # ipam: + # config: + # - subnet: 198.18.0.0/15 + # - gateway: 198.18.0.255 + volumes: data: config: diff --git a/docker.local/p0docker-compose.yml b/docker.local/p0docker-compose.yml index fbcbb45da..7c49f4bde 100644 --- a/docker.local/p0docker-compose.yml +++ b/docker.local/p0docker-compose.yml @@ -73,6 +73,11 @@ networks: driver: bridge testnet0: external: true + # driver: bridge + # ipam: + # config: + # - subnet: 198.18.0.0/15 + # - gateway: 198.18.0.255 volumes: data: From f57d57674315507046ae8df928e4c0bd7b48209e Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 23 Sep 2021 20:03:50 +0800 Subject: [PATCH 006/161] Fix/GitHub action checkout (#352) * fix(devops):used checkout@v1 * fix(devops):try run CodeQL on self-hosted * fix(devops):try run CodeQL on [self-hosted,build] * fix(devops):try run CodeQL on [self-hosted,docker-builds] * fix(devops):setup golang 1.16.x in CodeQL * fix(devops):use checkout@v1 in dockerize.yml * fix(devops):enabled dockerize.yml on any branch for testing * fix(devops):use checkout@v1 in integration-tests * fix(devops):test docker build on fix/github_action_checkout * fix(devops): fixed checkout issue on [self-hosted, build] * fix(devops): reverted integration-test to run on ubuntu-20.04 --- .github/workflows/ci.yml | 4 ++-- .github/workflows/codeql-analysis.yml | 12 +++++++++--- .github/workflows/dockerize.yml | 10 +++------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb4ecc7ef..5b7103566 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ env: jobs: integration_test: - runs-on: ubuntu-20.04 + runs-on: [ubuntu-20.04] timeout-minutes: 40 steps: @@ -26,7 +26,7 @@ jobs: go-version: '1.16' # The Go version to download (if necessary) and use. - name: Clone Blobber - uses: actions/checkout@v2 + uses: actions/checkout@v1 - name: Build test environment and run tests run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 216973208..d30a73171 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,14 +16,14 @@ on: branches: [ staging, master ] pull_request: # The branches below must be a subset of the branches above - branches: [ staging ] + branches: [ staging, master ] schedule: - cron: '43 17 * * 1' jobs: analyze: name: Analyze - runs-on: ubuntu-20.04 + runs-on: [self-hosted, build] permissions: actions: read contents: read @@ -39,7 +39,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v1 + + - name: Setup go 1.16 + if: success() + uses: actions/setup-go@v2 + with: + go-version: 1.16.x # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml index 0d5759db9..aee7d33c3 100644 --- a/.github/workflows/dockerize.yml +++ b/.github/workflows/dockerize.yml @@ -17,17 +17,13 @@ jobs: build_base: runs-on: [self-hosted, build] steps: - - name: Setup go 1.16 - uses: actions/setup-go@v2 - with: - go-version: '1.16' # The Go version to download (if necessary) and use. - name: Set GITHUB_ENV run: | echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber - uses: actions/checkout@v2 + uses: actions/checkout@v1 # - name: Clone gosdk # run: | @@ -62,7 +58,7 @@ jobs: echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber - uses: actions/checkout@v2 + uses: actions/checkout@v1 # - name: Clone gosdk # run: | @@ -142,7 +138,7 @@ jobs: echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber - uses: actions/checkout@v2 + uses: actions/checkout@v1 # - name: Clone gosdk # run: | From 463477b11d1434a4cf4a2ba8f6203b8435cf9358 Mon Sep 17 00:00:00 2001 From: Kush Trivedi <44091822+kushthedude@users.noreply.github.com> Date: Sat, 25 Sep 2021 14:28:15 +0530 Subject: [PATCH 007/161] Create dependabot.yml (#359) --- .github/dependabot.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..7d222cd0e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" + open-pull-requests-limit: 10 + reviewers: + - kushthedude + labels: + - pr/dependencies From d64e20b168cec63b22f510531004ef4bb5cf447f Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 25 Sep 2021 20:47:50 +0800 Subject: [PATCH 008/161] Fix/rate limit crash (#370) * fix(challenge): added recover in challenge worker * fix(ratelimit): upgraded tollbooth with v6 * fix(ratelimit): removed ratelimit from grpc-web * fix(ratelimit): reverted common.UserRateLimit * fix(ratelimit): fixed grcp test code * fix(ratelimit): use Middleware to limit request --- code/go/0chain.net/blobber/main.go | 2 +- .../blobbercore/challenge/challenge.go | 80 +++++++++- .../blobbercore/challenge/protocol.go | 3 +- .../blobbercore/challenge/worker.go | 146 ++---------------- .../blobbercore/handler/grpc_handler_test.go | 7 +- .../blobbercore/handler/grpc_middleware.go | 9 +- .../0chain.net/blobbercore/handler/handler.go | 72 +++++---- .../go/0chain.net/core/common/rate_limiter.go | 24 +-- code/go/0chain.net/core/transaction/http.go | 2 +- .../validatorcore/storage/handler.go | 5 +- go.mod | 2 +- go.sum | 6 +- 12 files changed, 165 insertions(+), 193 deletions(-) diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index f5ae1f061..10f449ffa 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -472,7 +472,7 @@ func main() { common.ConfigRateLimits() initHandlers(r) - grpcServer := handler.NewGRPCServerWithMiddlewares(common.NewGRPCRateLimiter(), r) + grpcServer := handler.NewGRPCServerWithMiddlewares(r) if config.Development() { reflection.Register(grpcServer) diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index c2c948a62..7bd02c41e 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -9,6 +9,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "github.com/remeh/sizedwaitgroup" @@ -23,11 +24,11 @@ type BCChallengeResponse struct { Challenges []*ChallengeEntity `json:"challenges"` } -// syncChallenges get challenge from blockchain , and add them in database -func syncChallenges(ctx context.Context) { +// syncOpenChallenges get challenge from blockchain , and add them in database +func syncOpenChallenges(ctx context.Context) { defer func() { if r := recover(); r != nil { - logging.Logger.Error("[recover] syncChallenges", zap.Any("err", r)) + logging.Logger.Error("[recover]challenge", zap.Any("err", r)) } }() @@ -95,11 +96,11 @@ func syncChallenges(ctx context.Context) { } } -// processChallenges read and process challenges from db -func processChallenges(ctx context.Context) { +// processAccepted read accepted challenge from db, and send them to validator to pass challenge +func processAccepted(ctx context.Context) { defer func() { if r := recover(); r != nil { - logging.Logger.Error("[recover] processChallenges", zap.Any("err", r)) + logging.Logger.Error("[recover]challenge", zap.Any("err", r)) } }() rctx := datastore.GetStore().CreateTransaction(ctx) @@ -119,7 +120,7 @@ func processChallenges(ctx context.Context) { go func(redeemCtx context.Context, challengeEntity *ChallengeEntity) { redeemCtx = datastore.GetStore().CreateTransaction(redeemCtx) defer redeemCtx.Done() - err := LoadValidationTickets(redeemCtx, challengeEntity) + err := loadValidationTickets(redeemCtx, challengeEntity) if err != nil { logging.Logger.Error("Getting validation tickets failed", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) } @@ -136,3 +137,68 @@ func processChallenges(ctx context.Context) { db.Rollback() rctx.Done() } + +// loadValidationTickets load validation tickets for challenge +func loadValidationTickets(ctx context.Context, challengeObj *ChallengeEntity) error { + mutex := lock.GetMutex(challengeObj.TableName(), challengeObj.ChallengeID) + mutex.Lock() + + defer func() { + if r := recover(); r != nil { + logging.Logger.Error("[recover] LoadValidationTickets", zap.Any("err", r)) + } + }() + + err := challengeObj.LoadValidationTickets(ctx) + if err != nil { + logging.Logger.Error("Error getting the validation tickets", zap.Error(err), zap.String("challenge_id", challengeObj.ChallengeID)) + } + + return err +} + +func commitProcessed(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + logging.Logger.Error("[recover]challenge", zap.Any("err", r)) + } + }() + + rctx := datastore.GetStore().CreateTransaction(ctx) + db := datastore.GetStore().GetTransaction(rctx) + openchallenges := make([]*ChallengeEntity, 0) + + db.Where(ChallengeEntity{Status: Processed}). + Order("sequence"). + Find(&openchallenges) + + for _, openchallenge := range openchallenges { + logging.Logger.Info("Attempting to commit challenge", zap.Any("challenge_id", openchallenge.ChallengeID), zap.Any("openchallenge", openchallenge)) + if err := openchallenge.UnmarshalFields(); err != nil { + logging.Logger.Error("ChallengeEntity_UnmarshalFields", zap.String("challenge_id", openchallenge.ChallengeID), zap.Error(err)) + } + mutex := lock.GetMutex(openchallenge.TableName(), openchallenge.ChallengeID) + mutex.Lock() + redeemCtx := datastore.GetStore().CreateTransaction(ctx) + err := openchallenge.CommitChallenge(redeemCtx, false) + if err != nil { + logging.Logger.Error("Error committing to blockchain", + zap.Error(err), + zap.String("challenge_id", openchallenge.ChallengeID)) + } + mutex.Unlock() + db := datastore.GetStore().GetTransaction(redeemCtx) + db.Commit() + if err == nil && openchallenge.Status == Committed { + logging.Logger.Info("Challenge has been submitted to blockchain", + zap.Any("id", openchallenge.ChallengeID), + zap.String("txn", openchallenge.CommitTxnID)) + } else { + logging.Logger.Info("Challenge was not committed", zap.Any("challenge_id", openchallenge.ChallengeID)) + break + } + } + + db.Rollback() + rctx.Done() +} diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 26b2b2c59..b578868b9 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -3,7 +3,6 @@ package challenge import ( "context" "encoding/json" - "fmt" "math" "math/rand" "time" @@ -183,7 +182,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { } url := validator.URL + VALIDATOR_URL - fmt.Println(url) + resp, err := util.SendPostRequest(url, postDataBytes, nil) if err != nil { Logger.Info("Got error from the validator.", zap.Any("error", err.Error())) diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index abc23e4e7..95ce9baa5 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -5,167 +5,51 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/core/lock" - - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "go.uber.org/zap" ) // SetupWorkers start challenge workers func SetupWorkers(ctx context.Context) { - go startSyncChallenges(ctx) - go startProcessChallenges(ctx) - go SubmitProcessedChallenges(ctx) //nolint:errcheck // goroutines + go startSyncOpen(ctx) + go startProcessAccepted(ctx) + go startCommitProcessed(ctx) } -// LoadValidationTickets load validation tickets for challenge -func LoadValidationTickets(ctx context.Context, challengeObj *ChallengeEntity) error { - mutex := lock.GetMutex(challengeObj.TableName(), challengeObj.ChallengeID) - mutex.Lock() - - defer func() { - if r := recover(); r != nil { - Logger.Error("[recover] LoadValidationTickets", zap.Any("err", r)) - } - }() - - err := challengeObj.LoadValidationTickets(ctx) - if err != nil { - Logger.Error("Error getting the validation tickets", zap.Error(err), zap.String("challenge_id", challengeObj.ChallengeID)) - } - - return err -} - -func SubmitProcessedChallenges(ctx context.Context) { +func startCommitProcessed(ctx context.Context) { + ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) + defer ticker.Stop() for { select { case <-ctx.Done(): return - default: - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) - - rows, err := db.Table("challenges"). - Select("commit_txn_id, sequence"). - Where(ChallengeEntity{Status: Committed}). - Order("sequence desc").Limit(1).Rows() - - if rows != nil && err == nil { - lastSeq := 0 - lastCommitTxn := "" - for rows.Next() { - if err := rows.Scan(&lastCommitTxn, &lastSeq); err != nil { - Logger.Error("Rows_Scan", zap.Error(err)) - } - } - - openchallenges := make([]*ChallengeEntity, 0) - - db.Where(ChallengeEntity{Status: Processed}). - Where("sequence > ?", lastSeq). - Order("sequence"). - Find(&openchallenges) - - if len(openchallenges) > 0 { - for _, openchallenge := range openchallenges { - Logger.Info("Attempting to commit challenge", zap.Any("challenge_id", openchallenge.ChallengeID), zap.Any("openchallenge", openchallenge)) - if err := openchallenge.UnmarshalFields(); err != nil { - Logger.Error("ChallengeEntity_UnmarshalFields", zap.String("challenge_id", openchallenge.ChallengeID), zap.Error(err)) - } - mutex := lock.GetMutex(openchallenge.TableName(), openchallenge.ChallengeID) - mutex.Lock() - redeemCtx := datastore.GetStore().CreateTransaction(ctx) - err := openchallenge.CommitChallenge(redeemCtx, false) - if err != nil { - Logger.Error("Error committing to blockchain", - zap.Error(err), - zap.String("challenge_id", openchallenge.ChallengeID)) - } - mutex.Unlock() - db := datastore.GetStore().GetTransaction(redeemCtx) - db.Commit() - if err == nil && openchallenge.Status == Committed { - Logger.Info("Challenge has been submitted to blockchain", - zap.Any("id", openchallenge.ChallengeID), - zap.String("txn", openchallenge.CommitTxnID)) - } else { - Logger.Info("Challenge was not committed", zap.Any("challenge_id", openchallenge.ChallengeID)) - break - } - } - } - db.Rollback() - rctx.Done() - - rctx = datastore.GetStore().CreateTransaction(ctx) - db = datastore.GetStore().GetTransaction(rctx) - toBeVerifiedChallenges := make([]*ChallengeEntity, 0) - // commit challenges on local state for all challenges that - // have missed the commit txn from blockchain - db.Where(ChallengeEntity{Status: Processed}). - Where("sequence < ?", lastSeq). - Find(&toBeVerifiedChallenges) - - for _, toBeVerifiedChallenge := range toBeVerifiedChallenges { - Logger.Info("Attempting to commit challenge through verification", zap.Any("challenge_id", toBeVerifiedChallenge.ChallengeID), zap.Any("openchallenge", toBeVerifiedChallenge)) - if err := toBeVerifiedChallenge.UnmarshalFields(); err != nil { - Logger.Error("ChallengeEntity_UnmarshalFields", zap.String("challenge_id", toBeVerifiedChallenge.ChallengeID), zap.Error(err)) - } - mutex := lock.GetMutex(toBeVerifiedChallenge.TableName(), toBeVerifiedChallenge.ChallengeID) - mutex.Lock() - redeemCtx := datastore.GetStore().CreateTransaction(ctx) - err := toBeVerifiedChallenge.CommitChallenge(redeemCtx, true) - if err != nil { - Logger.Error("Error committing to blockchain", - zap.Error(err), - zap.String("challenge_id", toBeVerifiedChallenge.ChallengeID)) - } - mutex.Unlock() - db := datastore.GetStore().GetTransaction(redeemCtx) - db.Commit() - if err == nil && toBeVerifiedChallenge.Status == Committed { - Logger.Info("Challenge has been submitted to blockchain", - zap.Any("id", toBeVerifiedChallenge.ChallengeID), - zap.String("txn", toBeVerifiedChallenge.CommitTxnID)) - } else { - Logger.Info("Challenge was not committed after verification", zap.Any("challenge_id", toBeVerifiedChallenge.ChallengeID)) - } - } - - db.Rollback() - rctx.Done() - } else { - Logger.Error("Error in getting the challenges for blockchain processing.", - zap.Error(err)) - } + case <-ticker.C: + commitProcessed(ctx) } - time.Sleep(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) } } -func startProcessChallenges(ctx context.Context) { +func startProcessAccepted(ctx context.Context) { ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) + defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: - processChallenges(ctx) + processAccepted(ctx) } } } -// startSyncChallenges -func startSyncChallenges(ctx context.Context) { +// startSyncOpen +func startSyncOpen(ctx context.Context) { ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) + defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: - syncChallenges(ctx) + syncOpenChallenges(ctx) } } } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go index 74a9031d3..e2e879e2b 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go @@ -3,15 +3,14 @@ package handler import ( "context" "errors" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "net" "regexp" "testing" "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" - rl "go.uber.org/ratelimit" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" @@ -33,7 +32,7 @@ var ( func startGRPCServer(t *testing.T) { lis = bufconn.Listen(1024 * 1024) - grpcS := NewGRPCServerWithMiddlewares(&common.GRPCRateLimiter{Limiter: rl.New(1000)}, mux.NewRouter()) + grpcS := NewGRPCServerWithMiddlewares(mux.NewRouter()) go func() { if err := grpcS.Serve(lis); err != nil { t.Errorf("Server exited with error: %v", err) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_middleware.go b/code/go/0chain.net/blobbercore/handler/grpc_middleware.go index 42097f244..f1e73d530 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_middleware.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_middleware.go @@ -13,7 +13,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/logging" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - grpc_ratelimit "github.com/grpc-ecosystem/go-grpc-middleware/ratelimit" grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" "go.uber.org/zap" "google.golang.org/grpc" @@ -58,7 +57,7 @@ func unaryTimeoutInterceptor() grpc.UnaryServerInterceptor { } } -func NewGRPCServerWithMiddlewares(limiter grpc_ratelimit.Limiter, r *mux.Router) *grpc.Server { +func NewGRPCServerWithMiddlewares(r *mux.Router) *grpc.Server { srv := grpc.NewServer( grpc.ChainStreamInterceptor( grpc_zap.StreamServerInterceptor(logging.Logger), @@ -68,7 +67,6 @@ func NewGRPCServerWithMiddlewares(limiter grpc_ratelimit.Limiter, r *mux.Router) grpc_zap.UnaryServerInterceptor(logging.Logger), grpc_recovery.UnaryServerInterceptor(), unaryDatabaseTransactionInjector(), - grpc_ratelimit.UnaryServerInterceptor(limiter), unaryTimeoutInterceptor(), // should always be the lastest, to be "innermost" ), ) @@ -79,6 +77,11 @@ func NewGRPCServerWithMiddlewares(limiter grpc_ratelimit.Limiter, r *mux.Router) wrappedServer := grpcweb.WrapServer(srv) r.Use(func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + logging.Logger.Error("[recover]grpc", zap.String("url", r.URL.String()), zap.Any("err", err)) + } + }() if wrappedServer.IsGrpcWebRequest(r) { wrappedServer.ServeHTTP(w, r) return diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 460bcd98c..da8ff943f 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -22,6 +22,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "github.com/gorilla/mux" @@ -35,40 +36,55 @@ func GetMetaDataStore() datastore.Store { /*SetupHandlers sets up the necessary API end points */ func SetupHandlers(r *mux.Router) { + + r.Use(panicRecovery, common.UseUserRateLimit) + //object operations - r.HandleFunc("/v1/file/upload/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(UploadHandler)))) - r.HandleFunc("/v1/file/download/{allocation}", common.UserRateLimit(common.ToByteStream(WithConnection(DownloadHandler)))).Methods("POST") - r.HandleFunc("/v1/file/rename/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(RenameHandler)))) - r.HandleFunc("/v1/file/copy/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CopyHandler)))) - r.HandleFunc("/v1/file/attributes/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(UpdateAttributesHandler)))) - r.HandleFunc("/v1/dir/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CreateDirHandler)))).Methods("POST") - r.HandleFunc("/v1/dir/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CreateDirHandler)))).Methods("DELETE") - r.HandleFunc("/v1/dir/rename/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CreateDirHandler)))).Methods("POST") - - r.HandleFunc("/v1/connection/commit/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CommitHandler)))) - r.HandleFunc("/v1/file/commitmetatxn/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CommitMetaTxnHandler)))) - r.HandleFunc("/v1/file/collaborator/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CollaboratorHandler)))) - r.HandleFunc("/v1/file/calculatehash/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CalculateHashHandler)))) + r.HandleFunc("/v1/file/upload/{allocation}", common.ToJSONResponse(WithConnection(UploadHandler))) + r.HandleFunc("/v1/file/download/{allocation}", common.ToByteStream(WithConnection(DownloadHandler))).Methods("POST") + r.HandleFunc("/v1/file/rename/{allocation}", common.ToJSONResponse(WithConnection(RenameHandler))) + r.HandleFunc("/v1/file/copy/{allocation}", common.ToJSONResponse(WithConnection(CopyHandler))) + r.HandleFunc("/v1/file/attributes/{allocation}", common.ToJSONResponse(WithConnection(UpdateAttributesHandler))) + r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("POST") + r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("DELETE") + r.HandleFunc("/v1/dir/rename/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("POST") + + r.HandleFunc("/v1/connection/commit/{allocation}", common.ToJSONResponse(WithConnection(CommitHandler))) + r.HandleFunc("/v1/file/commitmetatxn/{allocation}", common.ToJSONResponse(WithConnection(CommitMetaTxnHandler))) + r.HandleFunc("/v1/file/collaborator/{allocation}", common.ToJSONResponse(WithConnection(CollaboratorHandler))) + r.HandleFunc("/v1/file/calculatehash/{allocation}", common.ToJSONResponse(WithConnection(CalculateHashHandler))) //object info related apis - r.HandleFunc("/allocation", common.UserRateLimit(common.ToJSONResponse(WithConnection(AllocationHandler)))) - r.HandleFunc("/v1/file/meta/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(FileMetaHandler)))) - r.HandleFunc("/v1/file/stats/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(FileStatsHandler)))) - r.HandleFunc("/v1/file/list/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ListHandler)))) - r.HandleFunc("/v1/file/objectpath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler)))) - r.HandleFunc("/v1/file/referencepath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))) - r.HandleFunc("/v1/file/objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler)))) - r.HandleFunc("/v1/file/refs/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(RefsHandler)))).Methods("GET") + r.HandleFunc("/allocation", common.ToJSONResponse(WithConnection(AllocationHandler))) + r.HandleFunc("/v1/file/meta/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(FileMetaHandler))) + r.HandleFunc("/v1/file/stats/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(FileStatsHandler))) + r.HandleFunc("/v1/file/list/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ListHandler))) + r.HandleFunc("/v1/file/objectpath/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler))) + r.HandleFunc("/v1/file/referencepath/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler))) + r.HandleFunc("/v1/file/objecttree/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler))) + r.HandleFunc("/v1/file/refs/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(RefsHandler))).Methods("GET") //admin related - r.HandleFunc("/_debug", common.UserRateLimit(common.ToJSONResponse(DumpGoRoutines))) - r.HandleFunc("/_config", common.UserRateLimit(common.ToJSONResponse(GetConfig))) - r.HandleFunc("/_stats", common.UserRateLimit(stats.StatsHandler)) - r.HandleFunc("/_statsJSON", common.UserRateLimit(common.ToJSONResponse(stats.StatsJSONHandler))) - r.HandleFunc("/_cleanupdisk", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) - r.HandleFunc("/getstats", common.UserRateLimit(common.ToJSONResponse(stats.GetStatsHandler))) + r.HandleFunc("/_debug", common.ToJSONResponse(DumpGoRoutines)) + r.HandleFunc("/_config", common.ToJSONResponse(GetConfig)) + r.HandleFunc("/_stats", stats.StatsHandler) + r.HandleFunc("/_statsJSON", common.ToJSONResponse(stats.StatsJSONHandler)) + r.HandleFunc("/_cleanupdisk", common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler))) + r.HandleFunc("/getstats", common.ToJSONResponse(stats.GetStatsHandler)) //marketplace related - r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(MarketPlaceShareInfoHandler)))) + r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.ToJSONResponse(WithConnection(MarketPlaceShareInfoHandler))) +} + +func panicRecovery(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + logging.Logger.Error("[recover]http", zap.String("url", r.URL.String()), zap.Any("err", err)) + } + }() + + h.ServeHTTP(w, r) + }) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/core/common/rate_limiter.go b/code/go/0chain.net/core/common/rate_limiter.go index 6bb04168f..d1201be9d 100644 --- a/code/go/0chain.net/core/common/rate_limiter.go +++ b/code/go/0chain.net/core/common/rate_limiter.go @@ -4,8 +4,8 @@ import ( "net/http" "time" - "github.com/didip/tollbooth" - "github.com/didip/tollbooth/limiter" + "github.com/didip/tollbooth/v6" + "github.com/didip/tollbooth/v6/limiter" "github.com/spf13/viper" rl "go.uber.org/ratelimit" ) @@ -29,16 +29,10 @@ func (rl *ratelimit) init() { SetMethods([]string{"GET", "POST", "PUT", "DELETE"}) } -const DefaultRequestPerSecond = 100000 - //ConfigRateLimits - configure the rate limits func ConfigRateLimits() { userRl := viper.GetFloat64("handlers.rate_limit") - if userRl == 0 { - userRl = DefaultRequestPerSecond - } - userRateLimit = &ratelimit{RequestsPerSecond: userRl} userRateLimit.init() } @@ -46,10 +40,6 @@ func ConfigRateLimits() { func NewGRPCRateLimiter() *GRPCRateLimiter { userRl := viper.GetFloat64("handlers.rate_limit") - if userRl == 0 { - userRl = DefaultRequestPerSecond - } - return &GRPCRateLimiter{rl.New(int(userRl))} } @@ -64,6 +54,7 @@ func (r *GRPCRateLimiter) Limit() bool { //UserRateLimit - rate limiting for end user handlers func UserRateLimit(handler ReqRespHandlerf) ReqRespHandlerf { + if !userRateLimit.RateLimit { return handler } @@ -71,3 +62,12 @@ func UserRateLimit(handler ReqRespHandlerf) ReqRespHandlerf { tollbooth.LimitFuncHandler(userRateLimit.Limiter, handler).ServeHTTP(writer, request) } } + +//UserRateLimit - rate limiting for end user handlers +func UseUserRateLimit(h http.Handler) http.Handler { + if !userRateLimit.RateLimit { + return h + } + + return tollbooth.LimitFuncHandler(userRateLimit.Limiter, h.ServeHTTP) +} diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index c54cffc28..d610c4616 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -139,7 +139,7 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] resp.Body.Close() if err != nil { - Logger.Error("[sharder]"+resp.Status, zap.String("url", req.URL.String()), zap.String("response", string(resBody))) + Logger.Error("[sharder]"+err.Error(), zap.String("url", req.URL.String()), zap.String("response", string(resBody))) msgList = append(msgList, url+": "+err.Error()) return errors.Throw(ErrBadRequest, req.URL.String()+" "+err.Error()) diff --git a/code/go/0chain.net/validatorcore/storage/handler.go b/code/go/0chain.net/validatorcore/storage/handler.go index bfa73f0ae..52d7beac9 100644 --- a/code/go/0chain.net/validatorcore/storage/handler.go +++ b/code/go/0chain.net/validatorcore/storage/handler.go @@ -13,8 +13,9 @@ import ( /*SetupHandlers sets up the necessary API end points */ func SetupHandlers(r *mux.Router) { - r.HandleFunc("/v1/storage/challenge/new", common.UserRateLimit(common.ToJSONResponse(SetupContext(ChallengeHandler)))) - r.HandleFunc("/debug", common.UserRateLimit(common.ToJSONResponse(DumpGoRoutines))) + r.Use(common.UseUserRateLimit) + r.HandleFunc("/v1/storage/challenge/new", common.ToJSONResponse(SetupContext(ChallengeHandler))) + r.HandleFunc("/debug", common.ToJSONResponse(DumpGoRoutines)) } func DumpGoRoutines(ctx context.Context, r *http.Request) (interface{}, error) { diff --git a/go.mod b/go.mod index 751f34318..6045bde14 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/0chain/gosdk v1.2.88 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/didip/tollbooth v4.0.2+incompatible + github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 diff --git a/go.sum b/go.sum index 6e1914eef..165403c75 100644 --- a/go.sum +++ b/go.sum @@ -192,6 +192,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M= github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY= +github.com/didip/tollbooth/v6 v6.1.1 h1:Nt7PvWLa9Y94OrykXsFNBinVRQIu8xdy4avpl99Dc1M= +github.com/didip/tollbooth/v6 v6.1.1/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= @@ -251,6 +253,8 @@ github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc= +github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -668,7 +672,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -1062,6 +1065,7 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From b4966e75d3d8535b6a51a49075ded2e721fff0ad Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 1 Oct 2021 11:00:25 +0800 Subject: [PATCH 009/161] fix(dev): fixed cli --- dev.local/cli.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev.local/cli.sh b/dev.local/cli.sh index 06216cecb..cad782cbe 100755 --- a/dev.local/cli.sh +++ b/dev.local/cli.sh @@ -125,8 +125,8 @@ install_postgres () { cp -r ../sql "./data/blobber/" cd "./data/blobber/sql" - find . -name "*.sql" -exec sed -i '' "s/blobber_user/blobber_user/g" {} \; - find . -name "*.sql" -exec sed -i '' "s/blobber_meta/blobber_meta/g" {} \; + find . -name "*.sql" -exec sed -i '' "s/blobber_user/blobber_user$i/g" {} \; + find . -name "*.sql" -exec sed -i '' "s/blobber_meta/blobber_meta$i/g" {} \; cd $root From 2e4157df18fdaae0a570a7fc3c58330a2c66a867 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Fri, 1 Oct 2021 12:14:35 +0545 Subject: [PATCH 010/161] respond with certain level if level is present (#373) --- code/go/0chain.net/blobbercore/reference/referencepath.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index a0a81f209..396465e5d 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -130,7 +130,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, db1 = db1.Where("type = ?", _type) } if level != 0 { - db1 = db1.Where("level >= ?", level) + db1 = db1.Where("level = ?", level) } db1 = db1.Where("path > ?", offsetPath) @@ -147,7 +147,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, db2 = db2.Where("type = ?", _type) } if level != 0 { - db2 = db2.Where("level >= ?", level) + db2 = db2.Where("level = ?", level) } db2.Count(&totalRows) wg.Done() @@ -184,7 +184,7 @@ func GetUpdatedRefs(ctx context.Context, allocationID, path, offsetPath, _type, db1 = db1.Where("type = ?", _type) } if level != 0 { - db1 = db1.Where("level >= ?", level) + db1 = db1.Where("level = ?", level) } if updatedDate != "" { db1 = db1.Where("updated_at > ?", updatedDate) @@ -206,7 +206,7 @@ func GetUpdatedRefs(ctx context.Context, allocationID, path, offsetPath, _type, db2 = db2.Where("type > ?", level) } if level != 0 { - db2 = db2.Where("level >= ?", level) + db2 = db2.Where("level = ?", level) } if updatedDate != "" { db2 = db2.Where("updated_at > ?", updatedDate) From eda3e63fab349cf15fb1b28e2081a67eaffadb15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Oct 2021 14:37:32 +0800 Subject: [PATCH 011/161] chore(deps): bump go.uber.org/zap from 1.19.0 to 1.19.1 (#360) Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.19.0 to 1.19.1. - [Release notes](https://github.com/uber-go/zap/releases) - [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/zap/compare/v1.19.0...v1.19.1) --- updated-dependencies: - dependency-name: go.uber.org/zap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 6045bde14..c42db56ef 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 - go.uber.org/zap v1.19.0 + go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb google.golang.org/grpc v1.36.0 diff --git a/go.sum b/go.sum index 165403c75..6176756b4 100644 --- a/go.sum +++ b/go.sum @@ -827,6 +827,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= @@ -853,8 +854,9 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -868,8 +870,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -918,7 +921,6 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -972,6 +974,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1046,8 +1049,10 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1127,8 +1132,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 4a1102a5970907894852936aa8f8bf34c54f07e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Oct 2021 14:42:31 +0800 Subject: [PATCH 012/161] chore(deps): bump github.com/mitchellh/mapstructure from 1.3.1 to 1.4.2 (#363) Bumps [github.com/mitchellh/mapstructure](https://github.com/mitchellh/mapstructure) from 1.3.1 to 1.4.2. - [Release notes](https://github.com/mitchellh/mapstructure/releases) - [Changelog](https://github.com/mitchellh/mapstructure/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitchellh/mapstructure/compare/v1.3.1...v1.4.2) --- updated-dependencies: - dependency-name: github.com/mitchellh/mapstructure dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c42db56ef..ec288ff80 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jackc/pgproto3/v2 v2.0.4 // indirect github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible - github.com/mitchellh/mapstructure v1.3.1 + github.com/mitchellh/mapstructure v1.4.2 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce github.com/rs/cors v1.8.0 // indirect github.com/selvatico/go-mocket v1.0.7 diff --git a/go.sum b/go.sum index 6176756b4..d51f12f05 100644 --- a/go.sum +++ b/go.sum @@ -624,8 +624,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.1 h1:cCBH2gTD2K0OtLlv/Y5H01VQCqmlDxz30kS5Y5bqfLA= -github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 723f238d73efedec7799d1ca0f89a2ae1d2af5ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Oct 2021 14:51:02 +0800 Subject: [PATCH 013/161] chore(deps): bump github.com/spf13/viper from 1.7.0 to 1.9.0 (#361) Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.7.0 to 1.9.0. - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.7.0...v1.9.0) --- updated-dependencies: - dependency-name: github.com/spf13/viper dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Lz --- go.mod | 15 ++-- go.sum | 214 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 197 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index ec288ff80..a256b1388 100644 --- a/go.mod +++ b/go.mod @@ -22,16 +22,15 @@ require ( github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce github.com/rs/cors v1.8.0 // indirect github.com/selvatico/go-mocket v1.0.7 - github.com/spf13/viper v1.7.0 + github.com/spf13/viper v1.9.0 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 - go.uber.org/zap v1.19.1 - golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 - google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb - google.golang.org/grpc v1.36.0 - google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.1 - google.golang.org/protobuf v1.26.0 - gopkg.in/ini.v1 v1.61.0 // indirect + go.uber.org/zap v1.19.0 + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 + google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 + google.golang.org/grpc v1.40.0 + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 + google.golang.org/protobuf v1.27.1 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d diff --git a/go.sum b/go.sum index d51f12f05..42cf964e7 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,16 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -25,6 +35,7 @@ cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -91,6 +102,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -146,7 +158,9 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -162,6 +176,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -209,11 +224,15 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.10.8 h1:0UP5WUR8hh46ffbjJV7PK499+uGEyasRIfffS0vy06o= github.com/ethereum/go-ethereum v1.10.8/go.mod h1:pJNuIUYfX5+JKzSD/BTdNsvJSZ1TJqmz0dVyXMAbf6M= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= @@ -222,8 +241,9 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -277,6 +297,7 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -307,6 +328,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -321,8 +344,10 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= @@ -339,14 +364,18 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -354,6 +383,13 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -362,6 +398,7 @@ github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= @@ -383,24 +420,30 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaW github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0 h1:IvO4FbbQL6n3v3M1rQNobZ61SGL0gJLdvKA5KETM7Xs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0/go.mod h1:d2gYTOTUQklu06xp0AJYYmRdTVU1VKrqhkYfYag2L08= github.com/h2non/filetype v1.0.9 h1:Y9YFg/WJNd7XoC5h3WD+GZSxHmuRRDyJQ7fcIlIJplI= github.com/h2non/filetype v1.0.9/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -415,8 +458,11 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 h1:TV+x87PxW2BcEzpX6yWK0mBEGc9NFppsihFszD6QmyA= github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -429,6 +475,7 @@ github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/improbable-eng/grpc-web v0.14.0 h1:GdoK+cXABdB+1keuqsV1drSFO2XLYIxqt/4Rj8SWGBk= github.com/improbable-eng/grpc-web v0.14.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -517,8 +564,9 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -553,8 +601,10 @@ github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 h1:R7RAW1p8wjhlHKFhS4 github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20/go.mod h1:sh5SGGmQVGUkWDnxevz0I2FJ4TeC18hRPRjKVBMb2kA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -580,8 +630,9 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magma/augmented-networks/accounting/protos v0.1.1/go.mod h1:Hpfg8aAxldUN7qlVtR5xwlAf8pcetFm8DWwRKZsh2J4= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -589,6 +640,7 @@ github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIG github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= @@ -601,6 +653,8 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -613,9 +667,11 @@ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -675,8 +731,9 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= @@ -689,10 +746,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -735,6 +794,7 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -760,22 +820,26 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20201006035406-b97b5ead31f7/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= +github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -839,6 +903,9 @@ go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -848,15 +915,16 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -870,9 +938,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -887,6 +955,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -894,8 +963,9 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -922,6 +992,9 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -930,6 +1003,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -953,6 +1028,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -970,11 +1046,15 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -982,7 +1062,16 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -992,6 +1081,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1018,8 +1108,11 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1029,6 +1122,7 @@ golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1043,18 +1137,33 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1095,6 +1204,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1131,8 +1241,18 @@ golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1164,6 +1284,17 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1171,6 +1302,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1198,16 +1330,38 @@ google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb h1:hcskBH5qZCOa7WpTUFUFvoebnSFZBYpjykLtjIp9DVk= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 h1:z+ErRPu0+KS02Td3fOAgdX+lnPDh/VyaABEJPD4JRQs= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1225,13 +1379,24 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0-dev.0.20201218190559-666aea1fb34c/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.1 h1:M8spwkmx0pHrPq+uMdl22w5CvJ/Y+oAJTIs9oGoCpOE= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.1/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1245,8 +1410,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.25.1-0.20201208041424-160c7477e0e8/go.mod h1:hFxJC2f0epmp1elRCiEGJTKAWbwxZ2nvqZdHl3FQXCY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1261,8 +1427,8 @@ gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8 gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.61.0 h1:LBCdW4FmFYL4s/vDZD1RQYX7oAR6IjujCYgMdbHBR10= -gopkg.in/ini.v1 v1.61.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= From 1534da68bc4916d5a40c362321fe7d3bd0006c37 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 2 Oct 2021 08:14:45 +0800 Subject: [PATCH 014/161] Fix/gorm (#376) * fix(gomod): upgraded gorm and postgres driver * fix(gomod): downgraded gorm and postgres driver * fix(mux): remvoved grpc server wrapper * fix(mux): moved startGRPCServer for integration-tests only * fix(grpc):fixed syntax typo * fix(filestore):hide error if file doesn't exists on delete * fix(tests):fixed mock issue for gorm * fix(gomod):upgraded zap * fix(devops):updated reviewer for dependabot pr --- .github/dependabot.yml | 2 +- code/go/0chain.net/blobber/grpc.go | 32 +++++++++ code/go/0chain.net/blobber/main.go | 58 ++++------------- .../blobbercore/filestore/fs_store.go | 10 ++- .../blobbercore/handler/grpc_helpers.go | 10 +-- .../0chain.net/blobbercore/handler/handler.go | 15 +---- .../handler/handler_middlewares.go | 38 +++++++++++ .../blobbercore/handler/handler_test.go | 8 +-- .../handler/object_operation_handler_test.go | 2 +- dev.local/cli.sh | 16 +++-- dev.local/launch.json | 21 +++++- go.mod | 9 ++- go.sum | 65 ++++++++++++------- 13 files changed, 177 insertions(+), 109 deletions(-) create mode 100644 code/go/0chain.net/blobber/grpc.go create mode 100644 code/go/0chain.net/blobbercore/handler/handler_middlewares.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7d222cd0e..67fa2f678 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,6 +11,6 @@ updates: interval: "daily" open-pull-requests-limit: 10 reviewers: - - kushthedude + - cnlangzi labels: - pr/dependencies diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go new file mode 100644 index 000000000..298ad043e --- /dev/null +++ b/code/go/0chain.net/blobber/grpc.go @@ -0,0 +1,32 @@ +// +build !integration_tests + +package main + +import ( + "fmt" + "log" + "net" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/gorilla/mux" + "google.golang.org/grpc/reflection" +) + +func startGRPCServer(r mux.Router, port string) { + grpcServer := handler.NewGRPCServerWithMiddlewares(&r) + reflection.Register(grpcServer) + + if port == "" { + logging.Logger.Error("Could not start grpc server since grpc port has not been specified." + + " Please specify the grpc port in the --grpc_port build arguement to start the grpc server") + return + } + + logging.Logger.Info("listening too grpc requests on port - " + port) + lis, err := net.Listen("tcp", fmt.Sprintf(":%s", port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + log.Fatal(grpcServer.Serve(lis)) +} diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 10f449ffa..08927f196 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -6,15 +6,12 @@ import ( "fmt" "io" "log" - "net" "net/http" "os" "runtime" "strconv" "time" - "google.golang.org/grpc/reflection" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/challenge" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -33,7 +30,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "github.com/0chain/gosdk/zcncore" - "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/spf13/viper" "go.uber.org/zap" @@ -341,6 +337,9 @@ func setup(logDir string) error { // } func main() { + + grpcPortString := "" + deploymentMode := flag.Int("deployment_mode", 2, "deployment_mode") keysFile := flag.String("keys_file", "", "keys_file") minioFile := flag.String("minio_file", "", "minio_file") @@ -348,10 +347,11 @@ func main() { metadataDB = flag.String("db_dir", "", "db_dir") logDir := flag.String("log_dir", "", "log_dir") portString := flag.String("port", "", "port") - grpcPortString := flag.String("grpc_port", "", "grpc_port") hostname := flag.String("hostname", "", "hostname") configDir := flag.String("config_dir", "./config", "config_dir") + flag.StringVar(&grpcPortString, "grpc_port", "", "grpc_port") + flag.Parse() config.SetupDefaultConfig() @@ -453,39 +453,18 @@ func main() { var server *http.Server - // setup CORS r := mux.NewRouter() - headersOk := handlers.AllowedHeaders([]string{ - "X-Requested-With", "X-App-Client-ID", - "X-App-Client-Key", "Content-Type", - "X-App-Client-Signature", - }) - - // Allow anybody to access API. - // originsOk := handlers.AllowedOriginValidator(isValidOrigin) - originsOk := handlers.AllowedOrigins([]string{"*"}) - - methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", - "DELETE", "OPTIONS"}) - common.ConfigRateLimits() initHandlers(r) - grpcServer := handler.NewGRPCServerWithMiddlewares(r) - - if config.Development() { - reflection.Register(grpcServer) - } - - rHandler := handlers.CORS(originsOk, headersOk, methodsOk)(r) if config.Development() { // No WriteTimeout setup to enable pprof server = &http.Server{ Addr: address, ReadHeaderTimeout: 30 * time.Second, MaxHeaderBytes: 1 << 20, - Handler: rHandler, + Handler: r, } } else { server = &http.Server{ @@ -494,32 +473,19 @@ func main() { WriteTimeout: 30 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 1 << 20, - Handler: rHandler, + Handler: r, } } common.HandleShutdown(server) handler.HandleShutdown(common.GetRootContext()) Logger.Info("Ready to listen to the requests") - startTime = time.Now().UTC() - go func(gp *string) { - var grpcPort string - if gp != nil { - grpcPort = *gp - } - if grpcPort == "" { - Logger.Error("Could not start grpc server since grpc port has not been specified." + - " Please specify the grpc port in the --grpc_port build arguement to start the grpc server") - return - } + if config.Development() { + go startGRPCServer(*r, grpcPortString) + } + + startTime = time.Now().UTC() - Logger.Info("listening too grpc requests on port - " + grpcPort) - lis, err := net.Listen("tcp", fmt.Sprintf(":%s", grpcPort)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - log.Fatal(grpcServer.Serve(lis)) - }(grpcPortString) log.Fatal(server.ListenAndServe()) } diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index f17350927..a3223eb58 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -366,12 +366,18 @@ func (fs *FileFSStore) GetFileBlock(allocationID string, fileData *FileInputData func (fs *FileFSStore) DeleteTempFile(allocationID string, fileData *FileInputData, connectionID string) error { allocation, err := fs.SetupAllocation(allocationID, true) if err != nil { - return common.NewError("invalid_allocation", "Invalid allocation. "+err.Error()) + Logger.Warn("invalid_allocation", zap.String("allocationID", allocationID), zap.Error(err)) + return nil } fileObjectPath := fs.generateTempPath(allocation, fileData, connectionID) - return os.Remove(fileObjectPath) + err = os.Remove(fileObjectPath) + if err != nil { + Logger.Warn("invalid_path", zap.String("fileObjectPath", fileObjectPath), zap.Error(err)) + } + + return nil } func (fs *FileFSStore) generateTempPath(allocation *StoreAllocation, fileData *FileInputData, connectionID string) string { diff --git a/code/go/0chain.net/blobbercore/handler/grpc_helpers.go b/code/go/0chain.net/blobbercore/handler/grpc_helpers.go index 9f9889b4e..96257d1cd 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_helpers.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_helpers.go @@ -2,13 +2,14 @@ package handler import ( "context" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "net/http" + + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/gorilla/mux" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "google.golang.org/grpc" "google.golang.org/grpc/metadata" - "net/http" ) type gRPCHeaderMetadata struct { @@ -17,12 +18,11 @@ type gRPCHeaderMetadata struct { ClientSignature string } - func registerGRPCServices(r *mux.Router, server *grpc.Server) { blobberService := newGRPCBlobberService() grpcGatewayHandler := runtime.NewServeMux( - runtime.WithIncomingHeaderMatcher(CustomMatcher), - ) + runtime.WithIncomingHeaderMatcher(CustomMatcher), + ) blobbergrpc.RegisterBlobberServiceServer(server, blobberService) _ = blobbergrpc.RegisterBlobberServiceHandlerServer(context.Background(), grpcGatewayHandler, blobberService) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index da8ff943f..02f65d912 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -22,7 +22,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/logging" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "github.com/gorilla/mux" @@ -37,7 +36,7 @@ func GetMetaDataStore() datastore.Store { /*SetupHandlers sets up the necessary API end points */ func SetupHandlers(r *mux.Router) { - r.Use(panicRecovery, common.UseUserRateLimit) + r.Use(useRecovery, useCORS(), common.UseUserRateLimit) //object operations r.HandleFunc("/v1/file/upload/{allocation}", common.ToJSONResponse(WithConnection(UploadHandler))) @@ -75,18 +74,6 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.ToJSONResponse(WithConnection(MarketPlaceShareInfoHandler))) } -func panicRecovery(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - logging.Logger.Error("[recover]http", zap.String("url", r.URL.String()), zap.Any("err", err)) - } - }() - - h.ServeHTTP(w, r) - }) -} - func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { return func(ctx context.Context, r *http.Request) (interface{}, error) { ctx = GetMetaDataStore().CreateTransaction(ctx) diff --git a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go new file mode 100644 index 000000000..8ea009a4f --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go @@ -0,0 +1,38 @@ +package handler + +import ( + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/gorilla/handlers" + "go.uber.org/zap" +) + +func useCORS() func(http.Handler) http.Handler { + headersOk := handlers.AllowedHeaders([]string{ + "X-Requested-With", "X-App-Client-ID", + "X-App-Client-Key", "Content-Type", + "X-App-Client-Signature", + }) + + // Allow anybody to access API. + // originsOk := handlers.AllowedOriginValidator(isValidOrigin) + originsOk := handlers.AllowedOrigins([]string{"*"}) + + methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", + "DELETE", "OPTIONS"}) + + return handlers.CORS(originsOk, headersOk, methodsOk) +} + +func useRecovery(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + logging.Logger.Error("[recover]http", zap.String("url", r.URL.String()), zap.Any("err", err)) + } + }() + + h.ServeHTTP(w, r) + }) +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 7d0a3ecd2..5fdedbb3a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -1621,7 +1621,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { AddRow("/file.txt", "f", filePathHash, "abcd"), ) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(1) FROM "collaborators" WHERE`)). + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). WithArgs(client.GetClientID()). WillReturnError(gorm.ErrRecordNotFound) @@ -1634,7 +1634,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() - mock.ExpectExec(`UPDATE "read_markers"`). + mock.ExpectExec(`(UPDATE "read_markers" SET)(.+)`). WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) @@ -1736,7 +1736,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", "qCj3sXXeXUAByi1ERIbcfXzWN75dyocYzyRXnkStXio="), ) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(1) FROM "collaborators" WHERE`)). + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). WithArgs(client.GetClientID()). WillReturnError(gorm.ErrRecordNotFound) @@ -1868,7 +1868,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey()), ) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(1) FROM "collaborators" WHERE`)). + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). WithArgs(client.GetClientID()). WillReturnError(gorm.ErrRecordNotFound) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index 66c910ded..82a6b4e02 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -246,7 +246,7 @@ func TestDownloadFile(t *testing.T) { collaboratorRtv = 1 } mocket.Catcher.NewMock().OneTime().WithQuery( - `SELECT count(1) FROM "collaborators" WHERE`, + `SELECT count(*) FROM "collaborators" WHERE`, ).WithArgs( mockClient.ClientID, ).WithReply( diff --git a/dev.local/cli.sh b/dev.local/cli.sh index cad782cbe..9261f1528 100755 --- a/dev.local/cli.sh +++ b/dev.local/cli.sh @@ -97,15 +97,16 @@ cleanAll() { echo " " echo "Please select which blobber/validator you will work on: " -select i in "1" "2" "3" "clean all" "install debugers on .vscode/launch.json" "set hostname" "change zcn"; do +select i in "1" "2" "3" "integration-tests" "clean all" "install debugers on .vscode/launch.json" "set hostname" "change zcn"; do case $i in - "1" ) break;; - "2" ) break;; - "3" ) break;; - "clean all" ) cleanAll ;; + "1" ) break;; + "2" ) break;; + "3" ) break;; + "clean all" ) cleanAll ;; + "integration-tests" ) i="" && break ;; "install debugers on .vscode/launch.json" ) install_debuggger;; - "set hostname" ) set_hostname;; - "change zcn" ) change_zcn;; + "set hostname" ) set_hostname;; + "change zcn" ) change_zcn;; esac done @@ -171,6 +172,7 @@ prepareRuntime() { [ -d log ] || mkdir log } + start_blobber () { echo ">>>>>>>>>>>>>> Blobber $i <<<<<<<<<<<<<<<<" diff --git a/dev.local/launch.json b/dev.local/launch.json index 364b437d3..d29a15482 100644 --- a/dev.local/launch.json +++ b/dev.local/launch.json @@ -3,7 +3,26 @@ // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", - "configurations": [ + "configurations": [ + { + "name": "debug:integration-tests", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/code/go/0chain.net/blobber", + "args": ["--port","5051", + "--grpc_port", "31501", + "--hostname","Hostname", + "--deployment_mode", "0" , + "--keys_file", "../../../../docker.local/keys_config/b0bnode1_keys.txt", + "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt", + + "--files_dir","../../../../dev.local/data/blobber/files", + "--log_dir" ,"../../../../dev.local/data/blobber/log", + "--db_dir", "../../../../dev.local/data/blobber/data", + "--config_dir", "../../../../dev.local/data/blobber/config" + ] + }, { "name": "debug:blobber1", "type": "go", diff --git a/go.mod b/go.mod index a256b1388..b1b94ba9c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.14.0 - github.com/jackc/pgproto3/v2 v2.0.4 // indirect github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/mapstructure v1.4.2 @@ -25,8 +24,8 @@ require ( github.com/spf13/viper v1.9.0 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 - go.uber.org/zap v1.19.0 - golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 + go.uber.org/zap v1.19.1 + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 google.golang.org/grpc v1.40.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 @@ -34,8 +33,8 @@ require ( gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d - gorm.io/driver/postgres v1.0.0 - gorm.io/gorm v1.20.4 + gorm.io/driver/postgres v1.1.2 + gorm.io/gorm v1.21.15 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 42cf964e7..2cb96603a 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -266,6 +268,7 @@ github.com/go-ini/ini v1.55.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -503,12 +506,17 @@ github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5 github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= github.com/jackc/pgconn v1.6.1/go.mod h1:g8mKMqmSUO6AzAvha7vy07g1rbGOlc7iF0nU0ei83hc= -github.com/jackc/pgconn v1.6.4 h1:S7T6cx5o2OqmxdHaXLH1ZeD1SbI8jBznyYE9Ec0RCQ8= -github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= @@ -519,8 +527,9 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.0.4 h1:RHkX5ZUD9bl/kn0f9dYUWs1N7Nwvo1wwUYvKiR26Zco= -github.com/jackc/pgproto3/v2 v2.0.4/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= @@ -531,8 +540,9 @@ github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4 github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= github.com/jackc/pgtype v1.4.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= -github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY= -github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -540,12 +550,14 @@ github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXg github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.7.1/go.mod h1:nu42q3aPjuC1M0Nak4bnoprKlXPINqopEKqbq5AZSC4= -github.com/jackc/pgx/v4 v4.8.1 h1:SUbCLP2pXvf/Sr/25KsuI4aTxiFYIvpfk4l6aTSdyCw= -github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= @@ -554,8 +566,9 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jhump/protoreflect v1.8.1/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -622,8 +635,9 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= @@ -806,8 +820,9 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -923,8 +938,9 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -939,8 +955,9 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -962,10 +979,14 @@ golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -993,7 +1014,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1173,8 +1193,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1252,7 +1273,6 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1458,8 +1478,8 @@ gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d/go.mod h1:n2DTgk9at7cr/CWOT gorm.io/driver/mysql v0.3.1 h1:yvUT7Q0I3B9EHJ67NSp6cHbVwcdDHhVUsDAUiFFxRk0= gorm.io/driver/mysql v0.3.1/go.mod h1:A7H1JD9dKdcjeUTpTuWKEC+E1a74qzW7/zaXqKaTbfM= gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs= -gorm.io/driver/postgres v1.0.0 h1:Yh4jyFQ0a7F+JPU0Gtiam/eKmpT/XFc1FKxotGqc6FM= -gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to= +gorm.io/driver/postgres v1.1.2 h1:Amy3hCvLqM+/ICzjCnQr8wKFLVJTeOTdlMT7kCP+J1Q= +gorm.io/driver/postgres v1.1.2/go.mod h1:/AGV0zvqF3mt9ZtzLzQmXWQ/5vr+1V1TyHZGZVjzmwI= gorm.io/driver/sqlite v1.0.8 h1:omllgSb7/eh9D6lGvLZOdU1ZElxdXuO3dn3Rk+dQxUE= gorm.io/driver/sqlite v1.0.8/go.mod h1:xkm8/CEmA3yc4zRd0pdCqm43BjO8Hm6avfTpxWb/7c4= gorm.io/driver/sqlserver v0.2.5 h1:o/MXpn9/BB68RXEEQzfhsSL382yEqUtdCiGIuCspmkY= @@ -1467,9 +1487,8 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.20.4 h1:fMFR+3bdgx2/vf6VXFgNcsjUL3kSD7ioOFvby3PYTgE= -gorm.io/gorm v1.20.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.15 h1:gAyaDoPw0lCyrSFWhBlahbUA1U4P5RViC1uIqoB+1Rk= +gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From ef0aaddf2ad50a7301deab10462283a9779668c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Oct 2021 08:17:12 +0800 Subject: [PATCH 015/161] chore(deps): bump google.golang.org/grpc from 1.36.0 to 1.41.0 (#364) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.36.0 to 1.41.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.36.0...v1.41.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b1b94ba9c..68d09cca2 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 - google.golang.org/grpc v1.40.0 + google.golang.org/grpc v1.41.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.27.1 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect diff --git a/go.sum b/go.sum index 2cb96603a..5dec5bb60 100644 --- a/go.sum +++ b/go.sum @@ -163,6 +163,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -230,6 +231,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.10.8 h1:0UP5WUR8hh46ffbjJV7PK499+uGEyasRIfffS0vy06o= github.com/ethereum/go-ethereum v1.10.8/go.mod h1:pJNuIUYfX5+JKzSD/BTdNsvJSZ1TJqmz0dVyXMAbf6M= @@ -1412,8 +1414,9 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= From 991fb1cb16c04f74d5183c04663eb1ee7b73c4dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Oct 2021 08:56:50 +0800 Subject: [PATCH 016/161] chore(deps): bump github.com/improbable-eng/grpc-web (#366) Bumps [github.com/improbable-eng/grpc-web](https://github.com/improbable-eng/grpc-web) from 0.14.0 to 0.14.1. - [Release notes](https://github.com/improbable-eng/grpc-web/releases) - [Changelog](https://github.com/improbable-eng/grpc-web/blob/master/CHANGELOG.md) - [Commits](https://github.com/improbable-eng/grpc-web/compare/v0.14.0...v0.14.1) --- updated-dependencies: - dependency-name: github.com/improbable-eng/grpc-web dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 3 +-- go.sum | 32 +++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 68d09cca2..8dac20c6d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/0chain/errors v1.0.3 github.com/0chain/gosdk v1.2.88 github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect github.com/gorilla/handlers v1.5.1 @@ -14,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 - github.com/improbable-eng/grpc-web v0.14.0 + github.com/improbable-eng/grpc-web v0.14.1 github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/mapstructure v1.4.2 diff --git a/go.sum b/go.sum index 5dec5bb60..b01120582 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= @@ -421,6 +422,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -481,8 +483,8 @@ github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSa github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/improbable-eng/grpc-web v0.14.0 h1:GdoK+cXABdB+1keuqsV1drSFO2XLYIxqt/4Rj8SWGBk= -github.com/improbable-eng/grpc-web v0.14.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= +github.com/improbable-eng/grpc-web v0.14.1 h1:NrN4PY71A6tAz2sKDvC5JCauENWp0ykG8Oq1H3cpFvw= +github.com/improbable-eng/grpc-web v0.14.1/go.mod h1:zEjGHa8DAlkoOXmswrNvhUGEYQA9UI7DhrGeHR1DMGU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= @@ -575,11 +577,13 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -588,6 +592,7 @@ github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= @@ -616,6 +621,7 @@ github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 h1:R7RAW1p8wjhlHKFhS4 github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20/go.mod h1:sh5SGGmQVGUkWDnxevz0I2FJ4TeC18hRPRjKVBMb2kA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -705,8 +711,10 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -773,22 +781,28 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -829,6 +843,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -1002,6 +1018,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1058,6 +1075,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1140,6 +1158,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1151,11 +1170,14 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1364,6 +1386,7 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1402,6 +1425,7 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= @@ -1469,6 +1493,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -1502,6 +1527,7 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 3eacee9515f6f690f68025ffd733cd181e9d5f4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Oct 2021 09:22:25 +0800 Subject: [PATCH 017/161] chore(deps): bump github.com/grpc-ecosystem/grpc-gateway/v2 (#368) Bumps [github.com/grpc-ecosystem/grpc-gateway/v2](https://github.com/grpc-ecosystem/grpc-gateway) from 2.3.0 to 2.6.0. - [Release notes](https://github.com/grpc-ecosystem/grpc-gateway/releases) - [Changelog](https://github.com/grpc-ecosystem/grpc-gateway/blob/master/.goreleaser.yml) - [Commits](https://github.com/grpc-ecosystem/grpc-gateway/compare/v2.3.0...v2.6.0) --- updated-dependencies: - dependency-name: github.com/grpc-ecosystem/grpc-gateway/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 31 ++++++------------------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 8dac20c6d..b7bc5e2c3 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.14.1 github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 @@ -25,7 +25,7 @@ require ( go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 + google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 google.golang.org/grpc v1.41.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index b01120582..79c38aca9 100644 --- a/go.sum +++ b/go.sum @@ -124,7 +124,6 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= @@ -144,7 +143,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bufbuild/buf v0.37.0/go.mod h1:lQ1m2HkIaGOFba6w/aC3KYBHhKEOESP3gaAEpS3dAFM= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -185,7 +183,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= @@ -304,7 +301,6 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= @@ -320,8 +316,9 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -407,7 +404,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= @@ -430,8 +426,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0 h1:IvO4FbbQL6n3v3M1rQNobZ61SGL0gJLdvKA5KETM7Xs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0/go.mod h1:d2gYTOTUQklu06xp0AJYYmRdTVU1VKrqhkYfYag2L08= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 h1:rgxjzoDmDXw5q8HONgyHhBas4to0/XWRo/gPpJhsUNQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0/go.mod h1:qrJPVzv9YlhsrxJc3P/Q85nr0w1lIRikTl4JlhdDH5w= github.com/h2non/filetype v1.0.9 h1:Y9YFg/WJNd7XoC5h3WD+GZSxHmuRRDyJQ7fcIlIJplI= github.com/h2non/filetype v1.0.9/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -567,7 +563,6 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jhump/protoreflect v1.8.1/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -614,7 +609,6 @@ github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QH github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.9.11 h1:n2kipJFo+CPqg7fH988XJXjqEyj14RJ8BYj7UayxPNg= github.com/klauspost/reedsolomon v1.9.11/go.mod h1:nLvuzNvy1ZDNQW30IuMc2ZWCbiqrJgdLoUS2X8HAUVg= github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 h1:R7RAW1p8wjhlHKFhS4X7h8EePqADev/PltCmW9qlJoM= @@ -724,7 +718,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= @@ -769,7 +762,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -861,7 +853,6 @@ github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.1-0.20201006035406-b97b5ead31f7/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -901,7 +892,6 @@ github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefld github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/twitchtv/twirp v7.1.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= @@ -947,7 +937,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -971,7 +960,6 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= @@ -1279,10 +1267,8 @@ golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1387,9 +1373,7 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1405,8 +1389,9 @@ google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKr google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 h1:z+ErRPu0+KS02Td3fOAgdX+lnPDh/VyaABEJPD4JRQs= google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 h1:3V2dxSZpz4zozWWUq36vUxXEKnSYitEH2LdsAx+RUmg= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1429,7 +1414,6 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0-dev.0.20201218190559-666aea1fb34c/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= @@ -1441,7 +1425,6 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1454,8 +1437,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.25.1-0.20201208041424-160c7477e0e8/go.mod h1:hFxJC2f0epmp1elRCiEGJTKAWbwxZ2nvqZdHl3FQXCY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= From 67ae37de175af1d424d2d1ad3b93ae1e81f8c840 Mon Sep 17 00:00:00 2001 From: anish-squareops Date: Wed, 6 Oct 2021 11:33:32 +0000 Subject: [PATCH 018/161] added backup job --- .github/workflows/backup.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/backup.yml diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml new file mode 100644 index 000000000..9648f114e --- /dev/null +++ b/.github/workflows/backup.yml @@ -0,0 +1,28 @@ +name: Mirror repo to S3 +on: + schedule: + # Runs everyday at 12:25 am + - cron: '25 0 * * * ' + workflow_dispatch: +jobs: + s3Backup: + runs-on: docker-builds + steps: + - uses: actions/checkout@v1 + - name: Create backup + run: | + github-backup 0chain -P -t ${{ secrets.ACCESS_TOKEN }} --output-directory=/github-backup/blobber --all -O -R blobber + - name: Create zip + uses: papeloto/action-zip@v1 + with: + files: /github-backup/blobber + dest: blobber.zip + - name: Set AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Backup to s3 + run: | + aws s3 cp blobber.zip s3://${{ secrets.MIRROR_TARGET }}/blobber.zip \ No newline at end of file From ab56e7f51fd417f06bb75a10990cdc0d85c12a90 Mon Sep 17 00:00:00 2001 From: anish-squareops <68534207+anish-squareops@users.noreply.github.com> Date: Fri, 8 Oct 2021 13:08:04 +0530 Subject: [PATCH 019/161] backup (#383) --- .github/workflows/backup.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml index 9648f114e..f6b13beaa 100644 --- a/.github/workflows/backup.yml +++ b/.github/workflows/backup.yml @@ -13,10 +13,7 @@ jobs: run: | github-backup 0chain -P -t ${{ secrets.ACCESS_TOKEN }} --output-directory=/github-backup/blobber --all -O -R blobber - name: Create zip - uses: papeloto/action-zip@v1 - with: - files: /github-backup/blobber - dest: blobber.zip + run: zip -r blobber.zip /github-backup/blobber - name: Set AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From 9692b00d60605da2a34bf323a747f6063357488c Mon Sep 17 00:00:00 2001 From: "Mohd. Shahnawaz Ali Choudhary" <57221398+m-s-a-c@users.noreply.github.com> Date: Wed, 13 Oct 2021 18:16:39 +0530 Subject: [PATCH 020/161] Update and rename build.yml to build-&-publish-docker-image Update and rename build.yml to build-&-publish-docker-image --- .../workflows/{build.yml => build-&-publish-docker-image.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{build.yml => build-&-publish-docker-image.yml} (98%) diff --git a/.github/workflows/build.yml b/.github/workflows/build-&-publish-docker-image.yml similarity index 98% rename from .github/workflows/build.yml rename to .github/workflows/build-&-publish-docker-image.yml index ff57da5c9..2f33d0cd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -1,4 +1,4 @@ -name: docker-build +name: build-&-publish-docker-image on: workflow_dispatch: From 0c5de39b4adfa5b975a1b7e04a40c903c7726273 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Oct 2021 08:17:45 +0800 Subject: [PATCH 021/161] chore(deps): bump gorm.io/gorm from 1.21.15 to 1.21.16 (#384) Bumps [gorm.io/gorm](https://github.com/go-gorm/gorm) from 1.21.15 to 1.21.16. - [Release notes](https://github.com/go-gorm/gorm/releases) - [Commits](https://github.com/go-gorm/gorm/compare/v1.21.15...v1.21.16) --- updated-dependencies: - dependency-name: gorm.io/gorm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b7bc5e2c3..0ad973545 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.1.2 - gorm.io/gorm v1.21.15 + gorm.io/gorm v1.21.16 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 79c38aca9..b786db258 100644 --- a/go.sum +++ b/go.sum @@ -1496,8 +1496,9 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.21.15 h1:gAyaDoPw0lCyrSFWhBlahbUA1U4P5RViC1uIqoB+1Rk= gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.21.16 h1:YBIQLtP5PLfZQz59qfrq7xbrK7KWQ+JsXXCH/THlMqs= +gorm.io/gorm v1.21.16/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 16ffc02750e24162f7fe08cf17d99eedfe5c5795 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 15 Oct 2021 23:16:24 +0800 Subject: [PATCH 022/161] fix(copy):0chain/zboxcli:115 fixed hash issue on copy (#387) * fix(copy):0chain/zboxcli:115 fixed hash issue on copy * fix(devops): dont' run CodeQL and benchmark on pull request * fix(devops): prevent duplicated ci jobs on pull request * fix(devops): prevent duplicated ci jobs on pull request * fix(devops): enabled unit tests ci job on all pull request * fix(devops): enabled unit tests ci job on all pull request * fix(copy):#0chain/zboxcli:115 remove logging --- .github/workflows/benchmark.yml | 4 +- .github/workflows/ci.yml | 1 + .github/workflows/codeql-analysis.yml | 5 - .github/workflows/tests.yml | 6 +- .../blobbercore/allocation/copyfilechange.go | 1 + .../handler/file_command_update_bak.go | 138 ------------------ .../0chain.net/blobbercore/reference/ref.go | 16 +- 7 files changed, 10 insertions(+), 161 deletions(-) delete mode 100644 code/go/0chain.net/blobbercore/handler/file_command_update_bak.go diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b5616a4fd..66e35cf5a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,9 +2,7 @@ name: Benchmark tests on: push: - branches: [ master, staging, stream ] - pull_request: - branches: [ master, staging, stream ] + branches: [ master, staging ] jobs: benchmark: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b7103566..6c5090a16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI on: push: + branches: [ master, staging ] release: types: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d30a73171..252cc69ff 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,11 +12,6 @@ name: "CodeQL" on: - push: - branches: [ staging, master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ staging, master ] schedule: - cron: '43 17 * * 1' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39df09bb7..e30777b4a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,11 +5,7 @@ on: branches: - master - staging - - stream - pull_request: - - master - - staging - - stream + pull_request: jobs: unit_tests: diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index df52b209b..d655633fa 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -120,6 +120,7 @@ func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef *refe newFile.ActualThumbnailSize = affectedRef.ActualThumbnailSize newFile.EncryptedKey = affectedRef.EncryptedKey newFile.Attributes = datatypes.JSON(string(affectedRef.Attributes)) + newFile.ChunkSize = affectedRef.ChunkSize destRef.AddChild(newFile) } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update_bak.go b/code/go/0chain.net/blobbercore/handler/file_command_update_bak.go deleted file mode 100644 index 6bc66f690..000000000 --- a/code/go/0chain.net/blobbercore/handler/file_command_update_bak.go +++ /dev/null @@ -1,138 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "net/http" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/zboxcore/fileref" -) - -// UpdateFileCMD command for updating file -type UpdateFileCMD struct { - exisitingFileRef *reference.Ref - changeProcessor *allocation.UpdateFileChanger - allocationChange *allocation.AllocationChange -} - -// IsAuthorized validate request. -func (cmd *UpdateFileCMD) IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { - uploadMetaString := req.FormValue("updateMeta") - err := json.Unmarshal([]byte(uploadMetaString), &cmd.changeProcessor) - if err != nil { - return common.NewError("invalid_parameters", - "Invalid parameters. Error parsing the meta data for upload."+err.Error()) - } - - if cmd.changeProcessor.ChunkSize <= 0 { - cmd.changeProcessor.ChunkSize = fileref.CHUNK_SIZE - } - - cmd.exisitingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.changeProcessor.Path) - - if cmd.exisitingFileRef == nil { - return common.NewError("invalid_file_update", "File at path does not exist for update") - } - - if allocationObj.OwnerID != clientID && - allocationObj.RepairerID != clientID && - !reference.IsACollaborator(ctx, cmd.exisitingFileRef.ID, clientID) { - return common.NewError("invalid_operation", "Operation needs to be performed by the owner, collaborator or the payer of the allocation") - } - - return nil -} - -// ProcessContent flush file to FileStorage -func (cmd *UpdateFileCMD) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { - - result := blobberhttp.UploadResult{} - - origfile, _, err := req.FormFile("uploadFile") - if err != nil { - return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) - } - defer origfile.Close() - - fileInputData := &filestore.FileInputData{Name: cmd.changeProcessor.Filename, Path: cmd.changeProcessor.Path, OnCloud: cmd.exisitingFileRef.OnCloud} - fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) - if err != nil { - return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) - } - - result.Filename = cmd.changeProcessor.Filename - result.Hash = fileOutputData.ContentHash - result.MerkleRoot = fileOutputData.MerkleRoot - result.Size = fileOutputData.Size - - if len(cmd.changeProcessor.Hash) > 0 && cmd.changeProcessor.Hash != fileOutputData.ContentHash { - return result, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") - } - if len(cmd.changeProcessor.MerkleRoot) > 0 && cmd.changeProcessor.MerkleRoot != fileOutputData.MerkleRoot { - return result, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") - } - if fileOutputData.Size > config.Configuration.MaxFileSize { - return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") - } - - cmd.changeProcessor.Hash = fileOutputData.ContentHash - cmd.changeProcessor.MerkleRoot = fileOutputData.MerkleRoot - cmd.changeProcessor.AllocationID = allocationObj.ID - cmd.changeProcessor.Size = fileOutputData.Size - - allocationSize := fileOutputData.Size - - if allocationObj.BlobberSizeUsed+(allocationSize-cmd.exisitingFileRef.Size) > allocationObj.BlobberSize { - return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") - } - - cmd.allocationChange = &allocation.AllocationChange{} - cmd.allocationChange.ConnectionID = connectionObj.ConnectionID - cmd.allocationChange.Size = allocationSize - cmd.exisitingFileRef.Size - cmd.allocationChange.Operation = constants.FileOperationUpdate - - connectionObj.Size += cmd.allocationChange.Size - - return result, nil - -} - -// ProcessThumbnail flush thumbnail file to FileStorage if it has. -func (cmd *UpdateFileCMD) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { - - thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") - - if thumbHeader != nil { - - defer thumbfile.Close() - - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.changeProcessor.Path} - thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, thumbInputData, thumbfile, connectionObj.ConnectionID) - if err != nil { - return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) - } - if len(cmd.changeProcessor.ThumbnailHash) > 0 && cmd.changeProcessor.ThumbnailHash != thumbOutputData.ContentHash { - return common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") - } - cmd.changeProcessor.ThumbnailHash = thumbOutputData.ContentHash - cmd.changeProcessor.ThumbnailSize = thumbOutputData.Size - cmd.changeProcessor.ThumbnailFilename = thumbInputData.Name - } - - return nil - -} - -// UpdateChange add UpdateFileChanger in db -func (cmd *UpdateFileCMD) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { - connectionObj.AddChange(cmd.allocationChange, cmd.changeProcessor) - - return connectionObj.Save(ctx) -} diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index ce3fa0272..2531d3960 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -259,7 +259,7 @@ func (fr *Ref) GetFileHashData() string { if len(fr.Attributes) == 0 { fr.Attributes = datatypes.JSON("{}") } - hashArray := make([]string, 0) + hashArray := make([]string, 0, 11) hashArray = append(hashArray, fr.AllocationID) hashArray = append(hashArray, fr.Type) hashArray = append(hashArray, fr.Name) @@ -271,17 +271,15 @@ func (fr *Ref) GetFileHashData() string { hashArray = append(hashArray, fr.ActualFileHash) hashArray = append(hashArray, string(fr.Attributes)) hashArray = append(hashArray, strconv.FormatInt(fr.ChunkSize, 10)) + return strings.Join(hashArray, ":") } func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool) (string, error) { - // fmt.Println("fileref name , path, hash", fr.Name, fr.Path, fr.Hash) - // fmt.Println("Fileref hash data: " + fr.GetFileHashData()) fr.Hash = encryption.Hash(fr.GetFileHashData()) - // fmt.Println("Fileref hash : " + fr.Hash) fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize))) fr.PathHash = GetReferenceLookup(fr.AllocationID, fr.Path) - fr.PathLevel = len(GetSubDirsFromPath(fr.Path)) + 1 //strings.Count(fr.Path, "/") + fr.PathLevel = len(GetSubDirsFromPath(fr.Path)) + 1 fr.LookupHash = GetReferenceLookup(fr.AllocationID, fr.Path) var err error if saveToDB { @@ -314,15 +312,13 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (string, erro refNumBlocks += childRef.NumBlocks size += childRef.Size } - // fmt.Println("ref name and path, hash :" + r.Name + " " + r.Path + " " + r.Hash) - // fmt.Println("ref hash data: " + strings.Join(childHashes, ":")) + r.Hash = encryption.Hash(strings.Join(childHashes, ":")) - // fmt.Println("ref hash : " + r.Hash) + r.NumBlocks = refNumBlocks r.Size = size - //fmt.Println("Ref Path hash: " + strings.Join(childPathHashes, ":")) r.PathHash = encryption.Hash(strings.Join(childPathHashes, ":")) - r.PathLevel = len(GetSubDirsFromPath(r.Path)) + 1 //strings.Count(r.Path, "/") + r.PathLevel = len(GetSubDirsFromPath(r.Path)) + 1 r.LookupHash = GetReferenceLookup(r.AllocationID, r.Path) var err error From ef1b86d37924104b6b082a0d380bf1562a30240c Mon Sep 17 00:00:00 2001 From: Artem Bogomaz Date: Wed, 20 Oct 2021 07:40:06 +0700 Subject: [PATCH 023/161] :bug: Error sharing files with wrong clientID (#392) * :bug: error sharing files with wrong clientID * :bug: fixing unittests * :bug: removing duplicated check --- .../0chain.net/blobbercore/handler/handler.go | 7 ++++++- .../blobbercore/handler/handler_test.go | 20 +++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 02f65d912..1f2fe485f 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -447,6 +447,11 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { return nil, err } + // dummy, to avoid input and sql error + if len(authTicket.ClientID) != 64 || len(authTicket.OwnerID) != 64 { + return nil, common.NewError("share_info_insert", "Wrong ownerID or clientID") + } + shareInfo := reference.ShareInfo{ OwnerID: authTicket.OwnerID, ClientID: authTicket.ClientID, @@ -464,7 +469,7 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { err = reference.AddShareInfo(ctx, shareInfo) } if err != nil { - return nil, err + return nil, common.NewError("share_info_insert", "Unable to save share info") } resp := map[string]interface{}{ diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 5fdedbb3a..721b6be66 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -1125,7 +1125,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { body := bytes.NewBuffer(nil) formWriter := multipart.NewWriter(body) shareClientEncryptionPublicKey := "kkk" - shareClientID := "abcdefgh" + shareClientID := "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77" require.NoError(t, formWriter.WriteField("encryption_public_key", shareClientEncryptionPublicKey)) remotePath := "/file.txt" filePathHash := "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c" @@ -1188,12 +1188,12 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs("abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WithArgs("da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). WillReturnRows(sqlmock.NewRows([]string{})) aa := sqlmock.AnyArg() mock.ExpectExec(`INSERT INTO "marketplace_share_info"`). - WithArgs(client.GetClientID(), "abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c", "regenkey", aa, false, aa). + WithArgs(client.GetClientID(), "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c", "regenkey", aa, false, aa). WillReturnResult(sqlmock.NewResult(0, 0)) }, wantCode: http.StatusOK, @@ -1213,7 +1213,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { body := bytes.NewBuffer(nil) formWriter := multipart.NewWriter(body) shareClientEncryptionPublicKey := "kkk" - shareClientID := "abcdefgh" + shareClientID := "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77" require.NoError(t, formWriter.WriteField("encryption_public_key", shareClientEncryptionPublicKey)) remotePath := "/file.txt" filePathHash := "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c" @@ -1276,7 +1276,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs("abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WithArgs("da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). WillReturnRows( sqlmock.NewRows([]string{"client_id", "owner_id"}). AddRow("abcdefgh", "owner"), @@ -1284,7 +1284,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "marketplace_share_info"`). - WithArgs("regenkey", "kkk", false, aa, "abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WithArgs("regenkey", "kkk", false, aa, "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). WillReturnResult(sqlmock.NewResult(0, 1)) }, wantCode: http.StatusOK, @@ -1303,7 +1303,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { body := bytes.NewBuffer(nil) formWriter := multipart.NewWriter(body) - shareClientID := "abcdefgh" + shareClientID := "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77" remotePath := "/file.txt" require.NoError(t, formWriter.WriteField("refereeClientID", shareClientID)) @@ -1363,7 +1363,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ) mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). - WithArgs(true, "abcdefgh", filePathHash). + WithArgs(true, "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", filePathHash). WillReturnResult(sqlmock.NewResult(0, 1)) }, @@ -1383,7 +1383,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { body := bytes.NewBuffer(nil) formWriter := multipart.NewWriter(body) - shareClientID := "abcdefgh" + shareClientID := "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77" remotePath := "/file.txt" require.NoError(t, formWriter.WriteField("refereeClientID", shareClientID)) @@ -1443,7 +1443,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ) mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). - WithArgs(true, "abcdefgh", filePathHash). + WithArgs(true, "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", filePathHash). WillReturnResult(sqlmock.NewResult(0, 0)) }, From 97771a8ebf9816ba40c12f7fee30a8a48cfab47d Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 20 Oct 2021 08:50:12 +0800 Subject: [PATCH 024/161] fix(docs):build.blobber.sh has been split into 3 files (#396) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53e642a23..6e5ad72ed 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0. ``` -./docker.local/bin/build.blobber.sh +./docker.local/bin/build.blobber.base.sh && ./docker.local/bin/build.blobber.sh && ./docker.local/bin/build.validator.sh ``` From fd2592402e5c95d47f80ea4c3710730d0509c543 Mon Sep 17 00:00:00 2001 From: Lz Date: Mon, 25 Oct 2021 19:16:48 +0800 Subject: [PATCH 025/161] Fix/challenge log (#400) * fix(challenge): improved logging for challenge * fix(challenge): improved logging and fixed VerifyMerklePath * fix(gomod): fixed gosdk package for ci jobs * fix(gomod): fixed gosdk package --- .../blobbercore/allocation/newfilechange.go | 4 +- .../blobbercore/challenge/challenge.go | 35 +++++++------ .../blobbercore/challenge/entity.go | 4 ++ .../blobbercore/challenge/protocol.go | 49 ++++++++++++------- .../blobbercore/challenge/worker.go | 14 ++---- code/go/0chain.net/core/transaction/http.go | 24 ++++----- go.mod | 4 +- go.sum | 4 +- 8 files changed, 76 insertions(+), 62 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/newfilechange.go b/code/go/0chain.net/blobbercore/allocation/newfilechange.go index 23d33fbe8..0428f6409 100644 --- a/code/go/0chain.net/blobbercore/allocation/newfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/newfilechange.go @@ -22,9 +22,9 @@ type NewFileChange struct { //client side: Path string `json:"filepath" validation:"required"` //client side: - ActualHash string `json:"actual_hash,omitempty" validation:"required"` + ActualHash string `json:"actual_hash,omitempty" ` //client side: - ActualSize int64 `json:"actual_size,omitempty" validation:"required"` + ActualSize int64 `json:"actual_size,omitempty"` //client side: ActualThumbnailSize int64 `json:"actual_thumb_size"` //client side: diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index 7bd02c41e..dbe081921 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -9,6 +9,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" @@ -40,7 +41,7 @@ func syncOpenChallenges(ctx context.Context) { retBytes, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/openchallenges", params, chain.GetServerChain()) if err != nil { - logging.Logger.Error("Error getting the open challenges from the blockchain", zap.Error(err)) + logging.Logger.Error("[challenge]open: ", zap.Error(err)) } else { bytesReader := bytes.NewBuffer(retBytes) @@ -50,11 +51,11 @@ func syncOpenChallenges(ctx context.Context) { errd := d.Decode(&blobberChallenges) if errd != nil { - logging.Logger.Error("Error in unmarshal of the sharder response", zap.Error(errd)) + logging.Logger.Error("[challenge]json: ", zap.Error(errd)) } else { for _, challengeObj := range blobberChallenges.Challenges { if challengeObj == nil || len(challengeObj.ChallengeID) == 0 { - logging.Logger.Info("No challenge entity from the challenge map") + logging.Logger.Info("[challenge]open: No challenge entity from the challenge map") continue } @@ -69,7 +70,7 @@ func syncOpenChallenges(ctx context.Context) { if err != nil { if !errors.Is(err, gorm.ErrRecordNotFound) { - logging.Logger.Info("Error in load challenge entity from database ", zap.Error(err)) + logging.Logger.Error("[challenge]db: ", zap.Error(err)) continue } } @@ -78,13 +79,16 @@ func syncOpenChallenges(ctx context.Context) { isNextChallengeOnChain := latestChallenge == nil || latestChallenge.ChallengeID == challengeObj.PrevChallengeID if isFirstChallengeInDatabase || isNextChallengeOnChain { - logging.Logger.Info("Adding new challenge found from blockchain", zap.String("challenge", challengeObj.ChallengeID)) + logging.Logger.Info("[challenge]add: ", zap.String("challenge_id", challengeObj.ChallengeID)) challengeObj.Status = Accepted + challengeObj.CreatedAt = common.ToTime(challengeObj.Created) + challengeObj.UpdatedAt = challengeObj.CreatedAt + if err := challengeObj.Save(tx); err != nil { - logging.Logger.Error("ChallengeEntity_Save", zap.String("challenge_id", challengeObj.ChallengeID), zap.Error(err)) + logging.Logger.Error("[challenge]db: ", zap.String("challenge_id", challengeObj.ChallengeID), zap.Error(err)) } } else { - logging.Logger.Error("Challenge chain is not valid") + logging.Logger.Error("[challenge]Challenge chain is not valid") } } @@ -110,26 +114,29 @@ func processAccepted(ctx context.Context) { if len(openchallenges) > 0 { swg := sizedwaitgroup.New(config.Configuration.ChallengeResolveNumWorkers) for _, openchallenge := range openchallenges { - logging.Logger.Info("Processing the challenge", zap.Any("challenge_id", openchallenge.ChallengeID), zap.Any("openchallenge", openchallenge)) + logging.Logger.Info("[challenge]process: ", zap.String("challenge_id", openchallenge.ChallengeID)) err := openchallenge.UnmarshalFields() if err != nil { - logging.Logger.Error("Error unmarshaling challenge entity.", zap.Error(err)) + logging.Logger.Error("[challenge]json: ", zap.Error(err)) continue } swg.Add() go func(redeemCtx context.Context, challengeEntity *ChallengeEntity) { + defer swg.Done() redeemCtx = datastore.GetStore().CreateTransaction(redeemCtx) defer redeemCtx.Done() err := loadValidationTickets(redeemCtx, challengeEntity) if err != nil { - logging.Logger.Error("Getting validation tickets failed", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) + logging.Logger.Error("[challenge]validate: ", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) + return } db := datastore.GetStore().GetTransaction(redeemCtx) err = db.Commit().Error if err != nil { - logging.Logger.Error("Error commiting the readmarker redeem", zap.Error(err)) + logging.Logger.Error("[challenge]db: ", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) + return } - swg.Done() + }(ctx, openchallenge) } swg.Wait() @@ -145,13 +152,13 @@ func loadValidationTickets(ctx context.Context, challengeObj *ChallengeEntity) e defer func() { if r := recover(); r != nil { - logging.Logger.Error("[recover] LoadValidationTickets", zap.Any("err", r)) + logging.Logger.Error("[recover]LoadValidationTickets", zap.Any("err", r)) } }() err := challengeObj.LoadValidationTickets(ctx) if err != nil { - logging.Logger.Error("Error getting the validation tickets", zap.Error(err), zap.String("challenge_id", challengeObj.ChallengeID)) + logging.Logger.Error("[challenge]load: ", zap.String("challenge_id", challengeObj.ChallengeID), zap.Error(err)) } return err diff --git a/code/go/0chain.net/blobbercore/challenge/entity.go b/code/go/0chain.net/blobbercore/challenge/entity.go index 7b6fcd7a4..bea780f62 100644 --- a/code/go/0chain.net/blobbercore/challenge/entity.go +++ b/code/go/0chain.net/blobbercore/challenge/entity.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -76,6 +77,9 @@ type ChallengeEntity struct { ObjectPathString datatypes.JSON `json:"-" gorm:"column:object_path"` ObjectPath *reference.ObjectPath `json:"object_path" gorm:"-"` Created common.Timestamp `json:"created" gorm:"-"` + + CreatedAt time.Time `gorm:"created_at"` + UpdatedAt time.Time `gorm:"updated_at"` } func (ChallengeEntity) TableName() string { diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index b578868b9..e3f9a5b0f 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -63,19 +63,23 @@ func (cr *ChallengeEntity) SubmitChallengeToBC(ctx context.Context) (*transactio func (cr *ChallengeEntity) ErrorChallenge(ctx context.Context, err error) { cr.StatusMessage = err.Error() + cr.UpdatedAt = time.Now().UTC() + if err := cr.Save(ctx); err != nil { - Logger.Error("ChallengeEntity_Save", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) + Logger.Error("[challenge]db: ", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) } } // LoadValidationTickets load validation tickets func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { if len(cr.Validators) == 0 { - cr.StatusMessage = "No validators assigned to the challange" + cr.StatusMessage = "No validators assigned to the challenge" + cr.UpdatedAt = time.Now().UTC() + if err := cr.Save(ctx); err != nil { - Logger.Error("ChallengeEntity_Save", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) + Logger.Error("[challenge]db: ", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) } - return common.NewError("no_validators", "No validators assigned to the challange") + return common.NewError("no_validators", "No validators assigned to the challenge") } allocationObj, err := allocation.GetAllocationByID(ctx, cr.AllocationID) @@ -97,16 +101,19 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { r := rand.New(rand.NewSource(cr.RandomNumber)) blockNum = r.Int63n(rootRef.NumBlocks) blockNum = blockNum + 1 + cr.BlockNum = blockNum } else { - Logger.Error("Got a challenge for a blank allocation") + err = common.NewError("allocation_is_blank", "Got a challenge for a blank allocation") + cr.ErrorChallenge(ctx, err) + return err } - cr.BlockNum = blockNum if err != nil { cr.ErrorChallenge(ctx, err) return err } - Logger.Info("blockNum for challenge", zap.Any("rootRef.NumBlocks", rootRef.NumBlocks), zap.Any("blockNum", blockNum), zap.Any("challenge_id", cr.ChallengeID), zap.Any("random_seed", cr.RandomNumber)) + + Logger.Info("[challenge]rand: ", zap.Any("rootRef.NumBlocks", rootRef.NumBlocks), zap.Any("blockNum", blockNum), zap.Any("challenge_id", cr.ChallengeID), zap.Any("random_seed", cr.RandomNumber)) objectPath, err := reference.GetObjectPath(ctx, cr.AllocationID, blockNum) if err != nil { cr.ErrorChallenge(ctx, err) @@ -165,7 +172,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { postDataBytes, err := json.Marshal(postData) if err != nil { - Logger.Error("Error in marshalling the post data for validation. " + err.Error()) + Logger.Error("[db]form: " + err.Error()) cr.ErrorChallenge(ctx, err) return err } @@ -185,7 +192,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { resp, err := util.SendPostRequest(url, postDataBytes, nil) if err != nil { - Logger.Info("Got error from the validator.", zap.Any("error", err.Error())) + Logger.Info("[challenge]post: ", zap.Any("error", err.Error())) delete(responses, validator.ID) cr.ValidationTickets[i] = nil continue @@ -193,15 +200,15 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { var validationTicket ValidationTicket err = json.Unmarshal(resp, &validationTicket) if err != nil { - Logger.Info("Got error decoding from the validator response .", zap.Any("resp", string(resp)), zap.Any("error", err.Error())) + Logger.Error("[challenge]resp: ", zap.String("validator", validator.ID), zap.Any("resp", string(resp)), zap.Any("error", err.Error())) delete(responses, validator.ID) cr.ValidationTickets[i] = nil continue } - Logger.Info("Got response from the validator.", zap.Any("validator_response", validationTicket)) + Logger.Info("[challenge]resp: Got response from the validator.", zap.Any("validator_response", validationTicket)) verified, err := validationTicket.VerifySign() if err != nil || !verified { - Logger.Info("Validation ticket from validator could not be verified.") + Logger.Error("[challenge]ticket: Validation ticket from validator could not be verified.", zap.String("validator", validator.ID)) delete(responses, validator.ID) cr.ValidationTickets[i] = nil continue @@ -219,22 +226,25 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { if vt.Result { numSuccess++ } else { + Logger.Error("[challenge]ticket: "+vt.Message, zap.String("validator", vt.ValidatorID)) numFailure++ } numValidatorsResponded++ } } - Logger.Info("validator response stats", zap.Any("challenge_id", cr.ChallengeID), zap.Any("validator_responses", responses)) + Logger.Info("[challenge]validator response stats", zap.Any("challenge_id", cr.ChallengeID), zap.Any("validator_responses", responses)) if numSuccess > (len(cr.Validators)/2) || numFailure > (len(cr.Validators)/2) || numValidatorsResponded == len(cr.Validators) { if numSuccess > (len(cr.Validators) / 2) { cr.Result = ChallengeSuccess } else { cr.Result = ChallengeFailure - //Logger.Error("Challenge failed by the validators", zap.Any("block_num", cr.BlockNum), zap.Any("object_path", objectPath), zap.Any("challenge", cr)) + + Logger.Error("[challenge]validate: ", zap.String("challenge_id", cr.ChallengeID), zap.Any("block_num", cr.BlockNum), zap.Any("object_path", objectPath)) } cr.Status = Processed + cr.UpdatedAt = time.Now().UTC() } else { cr.ErrorChallenge(ctx, common.NewError("no_consensus_challenge", "No Consensus on the challenge result. Erroring out the challenge")) return common.NewError("no_consensus_challenge", "No Consensus on the challenge result. Erroring out the challenge") @@ -247,19 +257,20 @@ func (cr *ChallengeEntity) CommitChallenge(ctx context.Context, verifyOnly bool) if len(cr.LastCommitTxnIDs) > 0 { for _, lastTxn := range cr.LastCommitTxnIDs { - Logger.Info("Verifying the transaction : " + lastTxn) + Logger.Info("[challenge]commit: Verifying the transaction : " + lastTxn) t, err := transaction.VerifyTransaction(lastTxn, chain.GetServerChain()) if err == nil { cr.Status = Committed cr.StatusMessage = t.TransactionOutput cr.CommitTxnID = t.Hash + cr.UpdatedAt = time.Now().UTC() if err := cr.Save(ctx); err != nil { - Logger.Error("ChallengeEntity_Save", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) + Logger.Error("[challenge]db: ", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) } FileChallenged(ctx, cr.RefID, cr.Result, cr.CommitTxnID) return nil } - Logger.Error("Error verifying the txn from BC."+lastTxn, zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) + Logger.Error("[challenge]trans: Error verifying the txn from BC."+lastTxn, zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) } } @@ -272,14 +283,16 @@ func (cr *ChallengeEntity) CommitChallenge(ctx context.Context, verifyOnly bool) if t != nil { cr.CommitTxnID = t.Hash cr.LastCommitTxnIDs = append(cr.LastCommitTxnIDs, t.Hash) + cr.UpdatedAt = time.Now().UTC() } cr.ErrorChallenge(ctx, err) - Logger.Error("Error while submitting challenge to BC.", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) + Logger.Error("[challenge]submit: Error while submitting challenge to BC.", zap.String("challenge_id", cr.ChallengeID), zap.Error(err)) } else { cr.Status = Committed cr.StatusMessage = t.TransactionOutput cr.CommitTxnID = t.Hash cr.LastCommitTxnIDs = append(cr.LastCommitTxnIDs, t.Hash) + cr.UpdatedAt = time.Now().UTC() } err = cr.Save(ctx) FileChallenged(ctx, cr.RefID, cr.Result, cr.CommitTxnID) diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index 95ce9baa5..10e6ec5c9 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -15,26 +15,23 @@ func SetupWorkers(ctx context.Context) { } func startCommitProcessed(ctx context.Context) { - ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) - defer ticker.Stop() for { select { case <-ctx.Done(): return - case <-ticker.C: + case <-time.After(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second): commitProcessed(ctx) } } } func startProcessAccepted(ctx context.Context) { - ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) - defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticker.C: + case <-time.After(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second): processAccepted(ctx) } } @@ -42,13 +39,12 @@ func startProcessAccepted(ctx context.Context) { // startSyncOpen func startSyncOpen(ctx context.Context) { - ticker := time.NewTicker(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second) - defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticker.C: + case <-time.After(time.Duration(config.Configuration.ChallengeResolveFreq) * time.Second): syncOpenChallenges(ctx) } } diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index d610c4616..c6ea69311 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -2,10 +2,9 @@ package transaction import ( "context" - "crypto/sha1" "encoding/hex" + "hash/fnv" "math" - "strconv" "fmt" "io" @@ -16,13 +15,11 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/errors" "github.com/0chain/gosdk/core/resty" "github.com/0chain/gosdk/core/util" "github.com/0chain/gosdk/zcncore" - - "go.uber.org/zap" ) const TXN_SUBMIT_URL = "v1/transaction/put" @@ -122,26 +119,23 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] url := req.URL.String() if resp.StatusCode != http.StatusOK { - resBody, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() + errorMsg := "[sharder]" + resp.Status + ": " + url + msgList = append(msgList, errorMsg) - Logger.Error("[sharder]"+resp.Status, zap.String("url", req.URL.String()), zap.String("response", string(resBody))) - - msgList = append(msgList, url+": ["+strconv.Itoa(resp.StatusCode)+"] "+string(resBody)) - - return errors.Throw(ErrBadRequest, req.URL.String()+" "+resp.Status) + return errors.Throw(ErrBadRequest, errorMsg) } - hash := sha1.New() + hash := fnv.New32() //use fnv for better performance teeReader := io.TeeReader(resp.Body, hash) resBody, err := ioutil.ReadAll(teeReader) resp.Body.Close() if err != nil { - Logger.Error("[sharder]"+err.Error(), zap.String("url", req.URL.String()), zap.String("response", string(resBody))) - msgList = append(msgList, url+": "+err.Error()) - return errors.Throw(ErrBadRequest, req.URL.String()+" "+err.Error()) + errorMsg := "[sharder]body: " + url + " " + err.Error() + msgList = append(msgList, errorMsg) + return errors.Throw(ErrBadRequest, errorMsg) } diff --git a/go.mod b/go.mod index 0ad973545..751e0f50f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.2.88 + github.com/0chain/gosdk v1.3.0-beta.2 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect @@ -37,4 +37,4 @@ require ( nhooyr.io/websocket v1.8.7 // indirect ) -// replace github.com/0chain/gosdk => ../gosdk +//replace github.com/0chain/gosdk => ../gosdk diff --git a/go.sum b/go.sum index b786db258..9f3f7e3f4 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/0chain/errors v1.0.2/go.mod h1:5t76jLb56TKfg/K2VD+eUMmNZJ42QsIRI8KzWuztwU4= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.2.88 h1:rllzNvDvLP9iDXI6jYT9v3aBgeucGvb1uOpA9Af2AjQ= -github.com/0chain/gosdk v1.2.88/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= +github.com/0chain/gosdk v1.3.0-beta.2 h1:+exlUvGFp1iG2UF+3Hken4CEfoYdHsBav+70be096IY= +github.com/0chain/gosdk v1.3.0-beta.2/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= From d3946195d9aecfe89f99748de95c121dced711c7 Mon Sep 17 00:00:00 2001 From: Vishwajeet Takle <81675776+VishwajeetT22@users.noreply.github.com> Date: Wed, 27 Oct 2021 14:45:16 +0530 Subject: [PATCH 026/161] Fixed a Typo error in README.md file (#402) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e5ad72ed..187bb1c5f 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ block_worker: http://198.18.0.98:9091 -This works as a dns service, You need to know the above url for any network you want to connect, Just replace it in the above metioned file. +This works as a dns service, You need to know the above url for any network you want to connect, Just replace it in the above mentioned file. For example: If you want to connect to beta network From fd4e313b3aa80c63c2523821559e7cb900dae9d5 Mon Sep 17 00:00:00 2001 From: "Mohd. Shahnawaz Ali Choudhary" <57221398+m-s-a-c@users.noreply.github.com> Date: Wed, 27 Oct 2021 22:05:49 +0530 Subject: [PATCH 027/161] Update backup gitaction for condition check. --- .github/workflows/backup.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml index f6b13beaa..9b8459fca 100644 --- a/.github/workflows/backup.yml +++ b/.github/workflows/backup.yml @@ -1,25 +1,45 @@ name: Mirror repo to S3 + on: schedule: # Runs everyday at 12:25 am - cron: '25 0 * * * ' workflow_dispatch: + jobs: s3Backup: runs-on: docker-builds steps: - uses: actions/checkout@v1 + + - name: Get info + id: get_info + run: | + GIT_COMMIT_DATE="$((`git log -n 1 --date-order --all | grep Date | awk '{ print $4 }'`-1))" + YESTERDAY_DATE="$((`date | awk '{ print $3 }'`-1))" + echo ::set-output name=GIT_COMMIT_DATE::${GIT_COMMIT_DATE} + echo ::set-output name=YESTERDAY_DATE::${YESTERDAY_DATE} + echo $GIT_COMMIT_DATE + echo $YESTERDAY_DATE + - name: Create backup + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE run: | github-backup 0chain -P -t ${{ secrets.ACCESS_TOKEN }} --output-directory=/github-backup/blobber --all -O -R blobber + - name: Create zip + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE run: zip -r blobber.zip /github-backup/blobber + - name: Set AWS credentials + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Backup to s3 + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE run: | - aws s3 cp blobber.zip s3://${{ secrets.MIRROR_TARGET }}/blobber.zip \ No newline at end of file + aws s3 cp blobber.zip s3://${{ secrets.MIRROR_TARGET }}/blobber.zip From 638ddd097d5fce3435dc5c2606b0802b25331070 Mon Sep 17 00:00:00 2001 From: "Mohd. Shahnawaz Ali Choudhary" <57221398+m-s-a-c@users.noreply.github.com> Date: Thu, 28 Oct 2021 10:15:50 +0530 Subject: [PATCH 028/161] Commit date correction in backup.yml --- .github/workflows/backup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml index 9b8459fca..b98af1ec5 100644 --- a/.github/workflows/backup.yml +++ b/.github/workflows/backup.yml @@ -15,7 +15,7 @@ jobs: - name: Get info id: get_info run: | - GIT_COMMIT_DATE="$((`git log -n 1 --date-order --all | grep Date | awk '{ print $4 }'`-1))" + GIT_COMMIT_DATE="$((`git log -n 1 --date-order --all | grep Date | awk '{ print $4 }'`))" YESTERDAY_DATE="$((`date | awk '{ print $3 }'`-1))" echo ::set-output name=GIT_COMMIT_DATE::${GIT_COMMIT_DATE} echo ::set-output name=YESTERDAY_DATE::${YESTERDAY_DATE} From 3a980601c34cf42e7432600cf63d65bc23cc8db3 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 29 Oct 2021 06:36:39 +0800 Subject: [PATCH 029/161] fix(docker):# fixed alpine mirros issue (#414) --- README.md | 2 +- docker.local/base.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 187bb1c5f..9a8fd4f5a 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0. ``` -./docker.local/bin/build.blobber.base.sh && ./docker.local/bin/build.blobber.sh && ./docker.local/bin/build.validator.sh +./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh && ./docker.local/bin/build.validator.sh ``` diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index f39a5860c..0d9429d9d 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.1-alpine3.14 as blobber_base LABEL zchain="blobber" -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep From f26e15cd24a88e7e0962c1b34faf88db5fc467b4 Mon Sep 17 00:00:00 2001 From: hm90121 <65766301+hm90121@users.noreply.github.com> Date: Fri, 29 Oct 2021 04:14:57 +0530 Subject: [PATCH 030/161] blobber readme changes (#410) Co-authored-by: Lz --- README.md | 180 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 120 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 9a8fd4f5a..599a65055 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,8 @@ # Blobber Setup +This readme provide instructions on how to run blobber locally - - -## Table of Contents - - +## Table of Contents - [Initial Setup](#initial-setup) - [Directory Setup for Blobbers](#directory-setup-for-blobbers) @@ -25,62 +22,95 @@ ## Initial Setup - - -### Directory Setup for Blobbers +### Required OS and Software Dependencies - +- Linux (Tested on Ubuntu Desktop 20.02) +- MacOS +- Docker([Link](https://docs.docker.com/engine/install/) +- Docker Compose([Link](https://docs.docker.com/compose/install/)) -In the git/blobber run the following command +### Directory Setup for Blobbers +1. Clone the Blobber repository using the command +``` +git clone https://github.com/0chain/blobber.git +``` +2. In the git/blobber run the following command - ``` - ./docker.local/bin/blobber.init.setup.sh - ``` - - ## Building and Starting the Nodes - 1. Setup a network called testnet0 for each of these node containers to talk to each other. ``` - docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 - ``` -2. Go to git/blobber directory to build containers using (skip this step if you want to use existing docker images) +Note: Run all scripts as sudo - +2. Set up the block_worker URL +A block worker URL is a field in the `blobber/config/0chain_validator.yaml` and `blobber/config/0chain_blobber.yaml` configuration files that require the URL of bloockchain network you want to connect to.For testing purposes we will connect to the beta 0chain network and replace the default URL in blobber/config/0chain_validator.yaml and 0chain_blobber.yaml with the below-mentioned URL. ``` +<<<<<<< HEAD +block_worker: http://beta.0chain.net/dns +======= ./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh && ./docker.local/bin/build.validator.sh +>>>>>>> staging ``` -To link to local gosdk so that the changes are reflected on the blobber build please use the below command - +3. Go back to the blobber directory and build blobber containers using the scripts below ``` +./docker.local/bin/build.base.sh +./docker.local/bin/build.blobber.sh +./docker.local/bin/build.validator.sh +``` +Note: Run all scripts as sudo. +This would take few minutes. -$ ./docker.local/bin/build.blobber.dev.sh +To link to local gosdk so that the changes are reflected on the blobber build please use the below command(optional) ``` -For Apple M1 chip builds: +./docker.local/bin/build.blobber.dev.sh ``` +For Mac with Apple M1 chip use the following [guide](https://github.com/0chain/blobber/blob/staging/dev.local/README.md) to build and start blobbers. -./docker.local/bin/build.blobber.sh -m1 - -``` +Now register a Wallet using zboxcli to perform storage operations on blobbers.Build instructions for zbox are [here](https://github.com/0chain/zboxcli#installation-guides) -3. After building the container for blobber, go to Blobber1 directory (git/blobber/docker.local/blobber1) and run the container using +5. Verify whether Zbox has properly build by running the following command +``` +./zbox +``` +6. To register a wallet on Zbox to be used both by the blockchain and blobbers. Use the following Zbox command + ``` +./zbox register +``` +Successful Response: +``` +Wallet Registered +``` +7. Now navigate to the .zcn folder (this is created during zbox build) +``` +cd $HOME/.zcn/ +``` +8. Open the wallet.json file. It should be similar to the similar to the output below: +``` +{"client_id":"4af719e1fdb6244159f17922382f162387bae3708250cab6bc1c20cd85fb594c", +"client_key":"da1769bd0203b9c84dc19846ed94155b58d1ffeb3bbe35d38db5bf2fddf5a91c91b22bc7c89dd87e1f1fecbb17ef0db93517dd3886a64274997ea46824d2c119","keys":[{"public_key":"da1769bd0203b9c84dc19846ed94155b58d1ffeb3bbe35d38db5bf2fddf5a91c91b22bc7c89dd87e1f1fecbb17ef0db93517dd3886a64274997ea46824d2c1> +"private_key":"542f6be49108f52203ce75222601397aad32e554451371581ba0eca56b093d19"}],"mnemonics":"butter whisper wheat hope duck mention bird half wedding aim good regret maximum illegal much inch immune unlock resource congress drift> +"version":"1.0","date_created":"2021-09-09T20:22:56+05:30"} +``` +9. Copy the client_id value and paste it into blobbers and validators settings. The files can be found in `blobber/config` directory. +10. Open both the `blobber/config/0chain_validator.yaml` and `blobber/config/0chain_blobber.yaml` and edit the `delegate_wallet` value with your `client_id` value. + +11 Now run the blobbers by navigating into blobber directories for Blobber1 (git/blobber/docker.local/blobber1) and run the container using ``` # For locally build images @@ -94,80 +124,110 @@ For Apple M1 chip builds: ## Troubleshooting - - -1. Ensure the port mapping is all correct: - - +12. Ensure the port mapping is all correct: ``` - docker ps ``` - - - -This should display a container image blobber_blobber and should have the ports mapped like "0.0.0.0:5050->5050/tcp" - -2. When starting multiple blobbers, it could happen that blobbers are not being registered properly (not returned on `zbox ls-blobbers`). +This should display the container image blobber_blobber and should have the ports mapped like "0.0.0.0:5050->5050/tcp" + +13. Now check whether the blobber has registered to the blockchain by running the following zbox command + +``` +./zbox ls-blobbers +``` +In the response you should see the local blobbers mentioned with their urls for example http://198.18.0.91:5051 and http://198.18.0.92:5052 + +Sample Response: +``` +- id: 0bf5ae461d6474ca1bebba028ea57d646043bbfb6a4188348fd649f0deec5df2 + url: http://beta.0chain.net:31304 + used / total capacity: 14.0 GiB / 100.0 GiB + last_health_check: 1635347306 + terms: + read_price: 26.874 mZCN / GB + write_price: 26.874 mZCN / GB / time_unit + min_lock_demand: 0.1 + cct: 2m0s + max_offer_duration: 744h0m0s +- id: 7a90e6790bcd3d78422d7a230390edc102870fe58c15472073922024985b1c7d + url: http://198.18.0.92:5052 + used / total capacity: 0 B / 1.0 GiB + last_health_check: 1635347427 + terms: + read_price: 10.000 mZCN / GB + write_price: 100.000 mZCN / GB / time_unit + min_lock_demand: 0.1 + cct: 2m0s + max_offer_duration: 744h0m0s +- id: f65af5d64000c7cd2883f4910eb69086f9d6e6635c744e62afcfab58b938ee25 + url: http://198.18.0.91:5051 + used / total capacity: 0 B / 1.0 GiB + last_health_check: 1635347950 + terms: + read_price: 10.000 mZCN / GB + write_price: 100.000 mZCN / GB / time_unit + min_lock_demand: 0.1 + cct: 2m0s + max_offer_duration: 744h0m0s +- id: f8dc4aaf3bb32ae0f4ed575dd6931a42b75e546e07cb37a6e1c6aaf1225891c5 + url: http://beta.0chain.net:31305 + used / total capacity: 13.3 GiB / 100.0 GiB + last_health_check: 1635347346 + terms: + read_price: 26.874 mZCN / GB + write_price: 26.865 mZCN / GB / time_unit + min_lock_demand: 0.1 + cct: 2m0s + max_offer_duration: 744h0m0s +``` + +Note: When starting multiple blobbers, it could happen that blobbers are not being registered properly (not returned on `zbox ls-blobbers`). Blobber registration takes sometime and adding at least 5 second wait before starting the next blobber usually avoids the issue. -3. If unable to create new allocations as shown below. +14. Now you can create allocations on blobber and store files. -zbox example +Note: If unable to create new allocations as shown below. ``` -zbox newallocation --lock 0.5 +./zbox newallocation --lock 0.5 Error creating allocation: transaction_not_found: Transaction was not found on any of the sharders ``` -To fix this issue, the `delegate_wallet` configured on `config/0chain_blober.yaml` and `config/0chain_validator.yaml` must lock some tokens on the blobber. -Through zbox, it can be done like the example below. +To fix this issue you must lock some tokens on the blobber.Get the local blobber id using the `./zbox ls-blobbers` and use the following command ``` zbox sp-lock --blobber_id f65af5d64000c7cd2883f4910eb69086f9d6e6635c744e62afcfab58b938ee25 --tokens 0.5 ``` - - ## Connect to other network - - Your network connection depends on the block_worker url you give in the `config/0chain_blobber/validator.yaml` and `0chain_blobber.yaml` config file. - - - + ``` block_worker: http://198.18.0.98:9091 ``` - - This works as a dns service, You need to know the above url for any network you want to connect, Just replace it in the above mentioned file. -For example: If you want to connect to beta network - +For example: If you want to connect to test network - ``` -block_worker: https://beta.0chain.net/dns +block_worker: https://test.0chain.net/dns ``` - ## Miscellaneous - ### Cleanup - 1. Get rid of old unused docker resources: From 2471dcf801a0e4b5e0e4142d5c1003171f94fe49 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 29 Oct 2021 08:22:59 +0800 Subject: [PATCH 031/161] fix(git): fixed conflict --- README.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index 599a65055..b1a6d9720 100644 --- a/README.md +++ b/README.md @@ -54,13 +54,7 @@ Note: Run all scripts as sudo A block worker URL is a field in the `blobber/config/0chain_validator.yaml` and `blobber/config/0chain_blobber.yaml` configuration files that require the URL of bloockchain network you want to connect to.For testing purposes we will connect to the beta 0chain network and replace the default URL in blobber/config/0chain_validator.yaml and 0chain_blobber.yaml with the below-mentioned URL. ``` -<<<<<<< HEAD block_worker: http://beta.0chain.net/dns -======= - -./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh && ./docker.local/bin/build.validator.sh - ->>>>>>> staging ``` 3. Go back to the blobber directory and build blobber containers using the scripts below @@ -399,4 +393,4 @@ delete_local_copy: true delete_cloud_copy: true -``` +``` \ No newline at end of file From c43f56242f811cad8dfceeed84ab5be9684206dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Oct 2021 08:28:00 +0800 Subject: [PATCH 032/161] chore(deps): bump gorm.io/driver/postgres from 1.1.2 to 1.2.1 (#411) Bumps [gorm.io/driver/postgres](https://github.com/go-gorm/postgres) from 1.1.2 to 1.2.1. - [Release notes](https://github.com/go-gorm/postgres/releases) - [Commits](https://github.com/go-gorm/postgres/compare/v1.1.2...v1.2.1) --- updated-dependencies: - dependency-name: gorm.io/driver/postgres dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 751e0f50f..609c3f681 100644 --- a/go.mod +++ b/go.mod @@ -32,8 +32,8 @@ require ( gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d - gorm.io/driver/postgres v1.1.2 - gorm.io/gorm v1.21.16 + gorm.io/driver/postgres v1.2.1 + gorm.io/gorm v1.22.0 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 9f3f7e3f4..ef4b97702 100644 --- a/go.sum +++ b/go.sum @@ -1487,8 +1487,8 @@ gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d/go.mod h1:n2DTgk9at7cr/CWOT gorm.io/driver/mysql v0.3.1 h1:yvUT7Q0I3B9EHJ67NSp6cHbVwcdDHhVUsDAUiFFxRk0= gorm.io/driver/mysql v0.3.1/go.mod h1:A7H1JD9dKdcjeUTpTuWKEC+E1a74qzW7/zaXqKaTbfM= gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs= -gorm.io/driver/postgres v1.1.2 h1:Amy3hCvLqM+/ICzjCnQr8wKFLVJTeOTdlMT7kCP+J1Q= -gorm.io/driver/postgres v1.1.2/go.mod h1:/AGV0zvqF3mt9ZtzLzQmXWQ/5vr+1V1TyHZGZVjzmwI= +gorm.io/driver/postgres v1.2.1 h1:JDQKnF7MC51dgL09Vbydc5kl83KkVDlcXfSPJ+xhh68= +gorm.io/driver/postgres v1.2.1/go.mod h1:SHRZhu+D0tLOHV5qbxZRUM6kBcf3jp/kxPz2mYMTsNY= gorm.io/driver/sqlite v1.0.8 h1:omllgSb7/eh9D6lGvLZOdU1ZElxdXuO3dn3Rk+dQxUE= gorm.io/driver/sqlite v1.0.8/go.mod h1:xkm8/CEmA3yc4zRd0pdCqm43BjO8Hm6avfTpxWb/7c4= gorm.io/driver/sqlserver v0.2.5 h1:o/MXpn9/BB68RXEEQzfhsSL382yEqUtdCiGIuCspmkY= @@ -1496,9 +1496,8 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.21.16 h1:YBIQLtP5PLfZQz59qfrq7xbrK7KWQ+JsXXCH/THlMqs= -gorm.io/gorm v1.21.16/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.0 h1:mTO7Im+aAEqixqnWfmb2Z9FCLnrdoaESc1tUAwM4GNE= +gorm.io/gorm v1.22.0/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From f429b35084afe025bad16c8ed17ee835b0d3848f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Oct 2021 09:07:44 +0800 Subject: [PATCH 033/161] chore(deps): bump gorm.io/gorm from 1.21.16 to 1.22.2 (#412) Bumps [gorm.io/gorm](https://github.com/go-gorm/gorm) from 1.21.16 to 1.22.2. - [Release notes](https://github.com/go-gorm/gorm/releases) - [Commits](https://github.com/go-gorm/gorm/compare/v1.21.16...v1.22.2) --- updated-dependencies: - dependency-name: gorm.io/gorm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 609c3f681..ff114bc6a 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.2.1 - gorm.io/gorm v1.22.0 + gorm.io/gorm v1.22.2 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index ef4b97702..64b0f44dd 100644 --- a/go.sum +++ b/go.sum @@ -1496,8 +1496,9 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.22.0 h1:mTO7Im+aAEqixqnWfmb2Z9FCLnrdoaESc1tUAwM4GNE= gorm.io/gorm v1.22.0/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.2 h1:1iKcvyJnR5bHydBhDqTwasOkoo6+o4Ms5cknSt6qP7I= +gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From cd2a3061c15034d14e74b95fadba9f43612f4fe4 Mon Sep 17 00:00:00 2001 From: Rezoan Tamal <36768796+iamrz1@users.noreply.github.com> Date: Mon, 1 Nov 2021 04:54:21 +0600 Subject: [PATCH 034/161] Fixed missing build tag in dockerized images (#415) --- dev.local/cli.sh | 4 ++-- docker.aws/build.blobber/Dockerfile | 2 +- docker.aws/build.validator/Dockerfile | 2 +- docker.local/Dockerfile | 2 +- docker.local/Dockerfile.dev | 2 +- docker.local/IntegrationTestsBlobberDockerfile | 2 +- docker.local/ValidatorDockerfile | 2 +- docker.local/ValidatorDockerfile.dev | 2 +- docker.local/blobber.Dockerfile | 9 ++------- docker.local/validator.Dockerfile | 2 +- 10 files changed, 12 insertions(+), 17 deletions(-) diff --git a/dev.local/cli.sh b/dev.local/cli.sh index 9261f1528..6c7bd4b1b 100755 --- a/dev.local/cli.sh +++ b/dev.local/cli.sh @@ -179,7 +179,7 @@ start_blobber () { echo "[1/3] build blobber..." cd ../code/go/0chain.net/blobber - go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/blobber . + go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/blobber . echo "[2/3] setup runtime..." prepareRuntime; @@ -206,7 +206,7 @@ start_validator () { echo "[1/3] build validator..." cd ../code/go/0chain.net/validator - go build -v -tags "bn256 development" -gcflags="-N -l" -ldflags "-X 0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/validator . + go build -v -tags "bn256 development" -gcflags="-N -l" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/validator . echo "[2/3] setup runtime" prepareRuntime; diff --git a/docker.aws/build.blobber/Dockerfile b/docker.aws/build.blobber/Dockerfile index 2c61ea11e..8bbea0091 100644 --- a/docker.aws/build.blobber/Dockerfile +++ b/docker.aws/build.blobber/Dockerfile @@ -32,7 +32,7 @@ WORKDIR $SRC_DIR/go/0chain.net/blobber ARG image_tag ARG go_build_mode ARG go_bls_tag -RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X 0chain.net/core/build.BuildTag=${image_tag}" +RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.aws/build.validator/Dockerfile b/docker.aws/build.validator/Dockerfile index 84c964349..f7a96c457 100644 --- a/docker.aws/build.validator/Dockerfile +++ b/docker.aws/build.validator/Dockerfile @@ -32,7 +32,7 @@ WORKDIR $SRC_DIR/go/0chain.net/validator ARG image_tag ARG go_build_mode ARG go_bls_tag -RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X 0chain.net/core/build.BuildTag=${image_tag}" +RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/Dockerfile b/docker.local/Dockerfile index 193fff650..81a1f54c6 100644 --- a/docker.local/Dockerfile +++ b/docker.local/Dockerfile @@ -36,7 +36,7 @@ WORKDIR $SRC_DIR/code/go/0chain.net/blobber ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT -RUN go build -v -tags "bn256 development" -gcflags "all=-N -l" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN go build -v -tags "bn256 development" -gcflags "all=-N -l" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/Dockerfile.dev b/docker.local/Dockerfile.dev index 83ca7019a..a23bda758 100644 --- a/docker.local/Dockerfile.dev +++ b/docker.local/Dockerfile.dev @@ -35,7 +35,7 @@ WORKDIR $SRC_DIR/go/0chain.net/blobber ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT -RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/IntegrationTestsBlobberDockerfile b/docker.local/IntegrationTestsBlobberDockerfile index e7d0c1bb2..9a85f8ea9 100644 --- a/docker.local/IntegrationTestsBlobberDockerfile +++ b/docker.local/IntegrationTestsBlobberDockerfile @@ -28,7 +28,7 @@ ADD ./code/go/0chain.net $SRC_DIR/go/0chain.net WORKDIR $SRC_DIR/go/0chain.net/blobber -RUN go build -v -tags "bn256 development integration_tests" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN go build -v -tags "bn256 development integration_tests" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/ValidatorDockerfile b/docker.local/ValidatorDockerfile index f451e522f..efbfbf82b 100644 --- a/docker.local/ValidatorDockerfile +++ b/docker.local/ValidatorDockerfile @@ -34,7 +34,7 @@ ADD . $SRC_DIR WORKDIR $SRC_DIR/code/go/0chain.net/validator -RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/ValidatorDockerfile.dev b/docker.local/ValidatorDockerfile.dev index a574c9841..a0bbf4914 100644 --- a/docker.local/ValidatorDockerfile.dev +++ b/docker.local/ValidatorDockerfile.dev @@ -33,7 +33,7 @@ ADD ./blobber/code/go/0chain.net $SRC_DIR/go/0chain.net WORKDIR $SRC_DIR/go/0chain.net/validator -RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index 528599e4a..ebcac2754 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -1,8 +1,6 @@ FROM blobber_base as blobber_build - LABEL zchain="blobber" - ENV SRC_DIR=/0chain ENV GO111MODULE=on #ENV GOPROXY=https://goproxy.cn,direct @@ -12,14 +10,13 @@ ENV GO111MODULE=on COPY . $SRC_DIR # COPY ./gosdk /gosdk -RUN cd $SRC_DIR/ && go mod download - +RUN cd $SRC_DIR/ && go mod download WORKDIR $SRC_DIR/code/go/0chain.net/blobber ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT -RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 @@ -28,8 +25,6 @@ COPY --from=blobber_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ /usr/local/lib/ - ENV APP_DIR=/blobber WORKDIR $APP_DIR COPY --from=blobber_build /0chain/code/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber - diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index 58cbb9c4a..b17d0a22c 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -17,7 +17,7 @@ RUN cd $SRC_DIR/ && go mod download WORKDIR $SRC_DIR/code/go/0chain.net/validator -RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 From 7f02d77c9d157915be27c0558a16987268909858 Mon Sep 17 00:00:00 2001 From: Andika Demas Riyandi Date: Thu, 4 Nov 2021 19:17:13 +0700 Subject: [PATCH 035/161] add CGO_ENABLED=1 to the unit test (#418) * add CGO_ENABLED=1 to the unit test * add CGO_ENABLED=1 to test and build command * fixed timestamp issue fixed timestamp issue on TestWriterMarker_GethashData Co-authored-by: Lz --- .github/workflows/benchmark.yml | 2 +- .github/workflows/tests.yml | 2 +- Makefile | 4 ++-- .../validatorcore/storage/writemarker/entity_test.go | 2 +- dev.local/cli.sh | 6 ++---- docker.aws/build.blobber/Dockerfile | 2 +- docker.aws/build.validator/Dockerfile | 2 +- docker.local/Dockerfile | 2 +- docker.local/Dockerfile.dev | 2 +- docker.local/IntegrationTestsBlobberDockerfile | 2 +- docker.local/ValidatorDockerfile | 2 +- docker.local/ValidatorDockerfile.dev | 2 +- docker.local/blobber.Dockerfile | 2 +- docker.local/validator.Dockerfile | 2 +- test.sh | 2 +- 15 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 66e35cf5a..ef0100349 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,7 +31,7 @@ jobs: # git checkout $BRANCH - name: Run benchmark - run: go test -tags bn256 -benchmem -run="BenchmarkUploadFile*" -bench="BenchmarkUploadFile*" ./... | tee benchmark.txt + run: CGO_ENABLED=1 go test -tags bn256 -benchmem -run="BenchmarkUploadFile*" -bench="BenchmarkUploadFile*" ./... | tee benchmark.txt # Run `github-action-benchmark` action # - name: Push benchmark report diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e30777b4a..a91b0f5d0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,7 +35,7 @@ jobs: - name: Run tests run: | cd $GITHUB_WORKSPACE/code/go/0chain.net/ - go test -tags bn256 -race -coverprofile=coverage.txt -covermode=atomic ./... + CGO_ENABLED=1 go test -tags bn256 -race -coverprofile=coverage.txt -covermode=atomic ./... # - name: Codecov # run: | # cd $GITHUB_WORKSPACE/code/go/0chain.net/ diff --git a/Makefile b/Makefile index 6415b08e4..0ffa6ad29 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ .PHONY: test test: - go test -tags bn256 ./...; + CGO_ENABLED=1 go test -tags bn256 ./...; .PHONY: lint lint: @@ -22,7 +22,7 @@ lint: .PHONY: integration-tests integration-tests: - go test -tags bn256 ./... -args integration; + CGO_ENABLED=1 go test -tags bn256 ./... -args integration; diff --git a/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go b/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go index 15b613d76..20d9dbef3 100644 --- a/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go +++ b/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go @@ -18,7 +18,7 @@ func TestWriteMarker_GetHashData(t *testing.T) { wm, wallet, err := setupEntityTest(t) require.NoError(t, err) - want := fmt.Sprintf("%v:%v:%v:%v:%v:%v:%v", "alloc_root", "prev_alloc_root", "alloc_id", "blobber_id", wallet.ClientID, 1, common.Now()) + want := fmt.Sprintf("%v:%v:%v:%v:%v:%v:%v", "alloc_root", "prev_alloc_root", "alloc_id", "blobber_id", wallet.ClientID, 1, wm.Timestamp) got := wm.GetHashData() assert.Equal(t, want, got) } diff --git a/dev.local/cli.sh b/dev.local/cli.sh index 6c7bd4b1b..a6e026d51 100755 --- a/dev.local/cli.sh +++ b/dev.local/cli.sh @@ -179,7 +179,7 @@ start_blobber () { echo "[1/3] build blobber..." cd ../code/go/0chain.net/blobber - go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/blobber . + CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/blobber . echo "[2/3] setup runtime..." prepareRuntime; @@ -206,7 +206,7 @@ start_validator () { echo "[1/3] build validator..." cd ../code/go/0chain.net/validator - go build -v -tags "bn256 development" -gcflags="-N -l" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/validator . + CGO_ENABLED=1 go build -v -tags "bn256 development" -gcflags="-N -l" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=dev" -o $root/data/blobber$i/validator . echo "[2/3] setup runtime" prepareRuntime; @@ -247,5 +247,3 @@ select f in "install postgres" "start blobber" "start validator" "clean"; do "clean" ) clean; break;; esac done - - diff --git a/docker.aws/build.blobber/Dockerfile b/docker.aws/build.blobber/Dockerfile index 8bbea0091..38ba7d7e4 100644 --- a/docker.aws/build.blobber/Dockerfile +++ b/docker.aws/build.blobber/Dockerfile @@ -32,7 +32,7 @@ WORKDIR $SRC_DIR/go/0chain.net/blobber ARG image_tag ARG go_build_mode ARG go_bls_tag -RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" +RUN CGO_ENABLED=1 go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.aws/build.validator/Dockerfile b/docker.aws/build.validator/Dockerfile index f7a96c457..122aee4aa 100644 --- a/docker.aws/build.validator/Dockerfile +++ b/docker.aws/build.validator/Dockerfile @@ -32,7 +32,7 @@ WORKDIR $SRC_DIR/go/0chain.net/validator ARG image_tag ARG go_build_mode ARG go_bls_tag -RUN go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" +RUN CGO_ENABLED=1 go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/Dockerfile b/docker.local/Dockerfile index 81a1f54c6..17da9f577 100644 --- a/docker.local/Dockerfile +++ b/docker.local/Dockerfile @@ -36,7 +36,7 @@ WORKDIR $SRC_DIR/code/go/0chain.net/blobber ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT -RUN go build -v -tags "bn256 development" -gcflags "all=-N -l" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -gcflags "all=-N -l" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/Dockerfile.dev b/docker.local/Dockerfile.dev index a23bda758..d65dd778b 100644 --- a/docker.local/Dockerfile.dev +++ b/docker.local/Dockerfile.dev @@ -35,7 +35,7 @@ WORKDIR $SRC_DIR/go/0chain.net/blobber ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT -RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/IntegrationTestsBlobberDockerfile b/docker.local/IntegrationTestsBlobberDockerfile index 9a85f8ea9..a6a132008 100644 --- a/docker.local/IntegrationTestsBlobberDockerfile +++ b/docker.local/IntegrationTestsBlobberDockerfile @@ -28,7 +28,7 @@ ADD ./code/go/0chain.net $SRC_DIR/go/0chain.net WORKDIR $SRC_DIR/go/0chain.net/blobber -RUN go build -v -tags "bn256 development integration_tests" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development integration_tests" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/ValidatorDockerfile b/docker.local/ValidatorDockerfile index efbfbf82b..db2a19b6c 100644 --- a/docker.local/ValidatorDockerfile +++ b/docker.local/ValidatorDockerfile @@ -34,7 +34,7 @@ ADD . $SRC_DIR WORKDIR $SRC_DIR/code/go/0chain.net/validator -RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/ValidatorDockerfile.dev b/docker.local/ValidatorDockerfile.dev index a0bbf4914..ab2a10953 100644 --- a/docker.local/ValidatorDockerfile.dev +++ b/docker.local/ValidatorDockerfile.dev @@ -33,7 +33,7 @@ ADD ./blobber/code/go/0chain.net $SRC_DIR/go/0chain.net WORKDIR $SRC_DIR/go/0chain.net/validator -RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index ebcac2754..d658ffeb3 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -16,7 +16,7 @@ WORKDIR $SRC_DIR/code/go/0chain.net/blobber ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT -RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index b17d0a22c..62191b7a7 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -17,7 +17,7 @@ RUN cd $SRC_DIR/ && go mod download WORKDIR $SRC_DIR/code/go/0chain.net/validator -RUN go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: FROM golang:1.17.1-alpine3.14 diff --git a/test.sh b/test.sh index 2480a6b22..7c4f7416d 100644 --- a/test.sh +++ b/test.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -e -cd code/go/0chain.net; go test ./...; \ No newline at end of file +cd code/go/0chain.net; CGO_ENABLED=1 go test ./...; \ No newline at end of file From a82e06f5805e9fa60379b31e87b043801ebb5287 Mon Sep 17 00:00:00 2001 From: Artem Bogomaz Date: Fri, 5 Nov 2021 17:20:16 +0700 Subject: [PATCH 036/161] :bug: Root folder sharing not allowing to dig inside 2nd layer subfolders (#393) * :bug: resolving issue with sharing root folder * :bug: adding more unit tests --- .../blobbercore/handler/handler_test.go | 422 +++++++++++++++++- .../blobbercore/handler/storage_handler.go | 13 +- 2 files changed, 433 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 721b6be66..2de8f4d70 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "os" "regexp" + "strings" "testing" "time" @@ -272,7 +273,11 @@ func GetAuthTicketForEncryptedFile(allocationID string, remotePath string, fileH at.ClientID = clientID at.FileName = remotePath at.FilePathHash = fileHash - at.RefType = fileref.FILE + if strings.HasSuffix(remotePath, "/") { + at.RefType = fileref.DIRECTORY + } else { + at.RefType = fileref.FILE + } timestamp := int64(common.Now()) at.Expiration = timestamp + 7776000 at.Timestamp = timestamp @@ -1898,6 +1903,421 @@ func TestHandlers_Requiring_Signature(t *testing.T) { wantCode: http.StatusOK, wantBody: "", }, + { + name: "DownloadFile_Encrypted_InSharedFolder_Permission_Allowed_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + + remotePath := "/" + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) + + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) + authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + if err != nil { + t.Fatal(err) + } + require.NoError(t, formWriter.WriteField("auth_token", authTicket)) + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = client.GetClientID() + err = rm.Sign() + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + header := make([]byte, 2*1024) + copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/"), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnError(gorm.ErrRecordNotFound) + + rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/", "d", rootPathHash, rootPathHash, "content_hash", "", "."), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(client.GetClientID()), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(client.GetClientID(), rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, encscheme.GetEncryptedKey()), + ) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "", + }, + { + name: "DownloadFile_Encrypted_InSharedFolderSubdirectory_Permission_Allowed_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + + remotePath := "/folder1" + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1/subfolder1/file.txt") + require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) + + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) + authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + if err != nil { + t.Fatal(err) + } + require.NoError(t, formWriter.WriteField("auth_token", authTicket)) + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = client.GetClientID() + err = rm.Sign() + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + header := make([]byte, 2*1024) + copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1/subfolder1/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder1/subfolder1"), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnError(gorm.ErrRecordNotFound) + + rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "."), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(client.GetClientID()), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(client.GetClientID(), rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, encscheme.GetEncryptedKey()), + ) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "", + }, + { + name: "DownloadFile_Encrypted_InSharedFolder_WrongFilePath_Permission_Rejected_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + + remotePath := "/folder1" + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder2/subfolder1/file.txt") + require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) + + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) + authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + if err != nil { + t.Fatal(err) + } + require.NoError(t, formWriter.WriteField("auth_token", authTicket)) + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = client.GetClientID() + err = rm.Sign() + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + header := make([]byte, 2*1024) + copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder2/subfolder1/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder2/subfolder1"), + ) + + rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "/"), + ) + + }, + wantCode: http.StatusBadRequest, + wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: cannot verify auth ticket: invalid_parameters: Auth ticket is not valid for the resource being requested\"}\n\n", + }, } tests := append(positiveTests, negativeTests...) for _, test := range tests { diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index bed0323ea..7def312be 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -72,7 +72,18 @@ func (fsh *StorageHandler) verifyAuthTicket(ctx context.Context, authTokenString if err != nil { return false, err } - if refRequested.ParentPath != authTokenRef.Path && !strings.HasPrefix(refRequested.ParentPath, authTokenRef.Path+"/") { + + parentPath := refRequested.ParentPath + if !strings.HasPrefix(parentPath, "/") { + parentPath = "/" + parentPath + } + + authRefPath := authTokenRef.Path + if strings.HasPrefix(authRefPath, ".") || authRefPath == "/" { + authRefPath = "" + } + + if refRequested.ParentPath != authTokenRef.Path && !strings.HasPrefix(parentPath, authRefPath+"/") { return false, common.NewError("invalid_parameters", "Auth ticket is not valid for the resource being requested") } } From 9347bb24e5a5b09e720d0da9cd5ad89e23ae63bf Mon Sep 17 00:00:00 2001 From: Artem Bogomaz Date: Fri, 5 Nov 2021 17:21:12 +0700 Subject: [PATCH 037/161] :bug: Fixing issue with Encrypted Files (#409) * fix(download): fixed download file with authticket * fix(gomod): upgrade gosdk * fix(test): fixed mock issue * fix(test): fixed mock issue * :bug: fixing issue with enc files * :bug: temporary removing tests * :bug: resolving encrypted download issue Co-authored-by: Lz --- code/go/0chain.net/blobbercore/handler/handler_test.go | 9 +++++---- .../blobbercore/handler/object_operation_handler.go | 9 +++------ go.mod | 2 +- go.sum | 4 ++++ 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 2de8f4d70..fd934c5e5 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -1670,6 +1670,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { if err != nil { t.Fatal(err) } + require.NoError(t, formWriter.WriteField("auth_token", authTicket)) rm := &marker.ReadMarker{} rm.ClientID = client.GetClientID() @@ -1737,8 +1738,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, filePathHash). WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", "qCj3sXXeXUAByi1ERIbcfXzWN75dyocYzyRXnkStXio="), + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", "qCj3sXXeXUAByi1ERIbcfXzWN75dyocYzyRXnkStXio=", 65536), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). @@ -1869,8 +1870,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, filePathHash). WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey()), + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), 65536), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index e1bf32d2d..fbe9144ed 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -439,10 +439,7 @@ func (fsh *StorageHandler) DownloadFile( } } - if len(fileref.EncryptedKey) > 0 { - if authToken == nil { - return nil, errors.New("auth ticket is required to download encrypted file") - } + if len(fileref.EncryptedKey) > 0 && authToken != nil { // should not happen, just in case if shareInfo == nil { @@ -463,9 +460,9 @@ func (fsh *StorageHandler) DownloadFile( totalSize := len(respData) result := []byte{} - for i := 0; i < totalSize; i += reference.CHUNK_SIZE { + for i := 0; i < totalSize; i += int(fileref.ChunkSize) { encMsg := &zencryption.EncryptedMessage{} - chunkData := respData[i:int64(math.Min(float64(i+reference.CHUNK_SIZE), float64(totalSize)))] + chunkData := respData[i:int64(math.Min(float64(i+int(fileref.ChunkSize)), float64(totalSize)))] encMsg.EncryptedData = chunkData[(2 * 1024):] diff --git a/go.mod b/go.mod index ff114bc6a..21a4ca9db 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.3.0-beta.2 + github.com/0chain/gosdk v1.3.0-beta.3.0.20211028024808-1a15c1339eb2 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index 64b0f44dd..b636a7c97 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,10 @@ github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= github.com/0chain/gosdk v1.3.0-beta.2 h1:+exlUvGFp1iG2UF+3Hken4CEfoYdHsBav+70be096IY= github.com/0chain/gosdk v1.3.0-beta.2/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= +github.com/0chain/gosdk v1.3.0-beta.2.0.20211028024402-f93ca468ead1 h1:1BretsR6oIFEETv9LxiEnoSVT01BBYtqPSUam6NCzJI= +github.com/0chain/gosdk v1.3.0-beta.2.0.20211028024402-f93ca468ead1/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= +github.com/0chain/gosdk v1.3.0-beta.3.0.20211028024808-1a15c1339eb2 h1:PXk/ufOXPC7sUZaMvjOW70wsjJ/phtx4SwDMqqetarU= +github.com/0chain/gosdk v1.3.0-beta.3.0.20211028024808-1a15c1339eb2/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= From 591bdf480d0871e310f3bd5ffa7db3635930109f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 7 Nov 2021 20:15:30 +0530 Subject: [PATCH 038/161] chore(deps): bump github.com/improbable-eng/grpc-web (#419) Bumps [github.com/improbable-eng/grpc-web](https://github.com/improbable-eng/grpc-web) from 0.14.1 to 0.15.0. - [Release notes](https://github.com/improbable-eng/grpc-web/releases) - [Changelog](https://github.com/improbable-eng/grpc-web/blob/master/CHANGELOG.md) - [Commits](https://github.com/improbable-eng/grpc-web/compare/v0.14.1...v0.15.0) --- updated-dependencies: - dependency-name: github.com/improbable-eng/grpc-web dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 21a4ca9db..7812bcd98 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 - github.com/improbable-eng/grpc-web v0.14.1 + github.com/improbable-eng/grpc-web v0.15.0 github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/mapstructure v1.4.2 diff --git a/go.sum b/go.sum index b636a7c97..97dd3bec1 100644 --- a/go.sum +++ b/go.sum @@ -149,7 +149,10 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -483,8 +486,8 @@ github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSa github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/improbable-eng/grpc-web v0.14.1 h1:NrN4PY71A6tAz2sKDvC5JCauENWp0ykG8Oq1H3cpFvw= -github.com/improbable-eng/grpc-web v0.14.1/go.mod h1:zEjGHa8DAlkoOXmswrNvhUGEYQA9UI7DhrGeHR1DMGU= +github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= +github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= @@ -1197,6 +1200,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From c5f2960fea9aae011fe99c446d8c585c958b8db3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 7 Nov 2021 20:16:07 +0530 Subject: [PATCH 039/161] chore(deps): bump google.golang.org/grpc from 1.41.0 to 1.42.0 (#416) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.41.0 to 1.42.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.41.0...v1.42.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7812bcd98..a254994d0 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 - google.golang.org/grpc v1.41.0 + google.golang.org/grpc v1.42.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.27.1 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect diff --git a/go.sum b/go.sum index 97dd3bec1..494115773 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,11 @@ github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3h github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -1431,8 +1434,8 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From 6e76d29148e1426d8001cf18da91c2e0a5974b94 Mon Sep 17 00:00:00 2001 From: anish-squareops <68534207+anish-squareops@users.noreply.github.com> Date: Mon, 8 Nov 2021 12:28:34 +0530 Subject: [PATCH 040/161] Update build-&-publish-docker-image.yml --- .github/workflows/build-&-publish-docker-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 2f33d0cd4..d0ccae868 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -1,6 +1,8 @@ name: build-&-publish-docker-image on: + push: + branches: [ staging ] workflow_dispatch: inputs: latest_tag: From 046d4e661db513c9cacec6bb3507a78da57f115b Mon Sep 17 00:00:00 2001 From: Rezoan Tamal <36768796+iamrz1@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:42:55 +0600 Subject: [PATCH 041/161] Signature verification in file meta api to ensure authorized access to allocation owner or a registered collaborator (#394) * Allocation signature verificatin for owner and collaborators in file-meta handler * fix(gomod): upgraded gosdk with latest commit * Allocation signature verificatin for owner and collaborators in file-meta handler Co-authored-by: Lz --- .../getfilemetadata_integration_test.go | 20 +- .../handler/helper_integration_test.go | 4 +- .../blobbercore/handler/storage_handler.go | 41 +- go.mod | 4 +- go.sum | 1455 +---------------- 5 files changed, 44 insertions(+), 1480 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go b/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go index 049670fa2..f5dfef241 100644 --- a/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go @@ -2,21 +2,25 @@ package handler import ( "context" - "testing" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "google.golang.org/grpc/metadata" + "testing" ) func TestGetFileMetaData_IntegrationTest(t *testing.T) { bClient, tdController := setupHandlerIntegrationTests(t) + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) err := tdController.ClearDatabase() if err != nil { t.Fatal(err) } - err = tdController.AddGetFileMetaDataTestData() + err = tdController.AddGetFileMetaDataTestData(allocationTx, pubKey) if err != nil { t.Fatal(err) } @@ -31,12 +35,13 @@ func TestGetFileMetaData_IntegrationTest(t *testing.T) { { name: "Success", context: metadata.New(map[string]string{ - common.ClientHeader: "exampleOwnerId", + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, }), input: &blobbergrpc.GetFileMetaDataRequest{ Path: "examplePath", PathHash: "exampleId:examplePath", - Allocation: "exampleTransaction", + Allocation: allocationTx, }, expectedFileName: "filename", expectingError: false, @@ -44,12 +49,13 @@ func TestGetFileMetaData_IntegrationTest(t *testing.T) { { name: "Unknown file path", context: metadata.New(map[string]string{ - common.ClientHeader: "exampleOwnerId", + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, }), input: &blobbergrpc.GetFileMetaDataRequest{ Path: "examplePath", PathHash: "exampleId:examplePath123", - Allocation: "exampleTransaction", + Allocation: allocationTx, }, expectedFileName: "", expectingError: true, diff --git a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go index 3a744b396..fc03787af 100644 --- a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go @@ -199,7 +199,7 @@ VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKe return nil } -func (c *TestDataController) AddGetFileMetaDataTestData() error { +func (c *TestDataController) AddGetFileMetaDataTestData(allocationTx, pubKey string) error { var err error var tx *sql.Tx defer func() { @@ -227,7 +227,7 @@ func (c *TestDataController) AddGetFileMetaDataTestData() error { _, err = tx.Exec(` INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id, repairer_id, is_immutable) -VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId', 'repairer_id', false); +VALUES ('exampleId' ,'` + allocationTx + `','exampleOwnerId','` + pubKey + `',` + fmt.Sprint(expTime) + `,'examplePayerId', 'repairer_id', false); `) if err != nil { return err diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 7def312be..bda20ebb4 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -133,9 +133,9 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") } + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) alloc, err := fsh.verifyAllocation(ctx, allocationTx, true) - if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } @@ -146,19 +146,34 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - _ = ctx.Value(constants.ContextKeyClientKey).(string) - pathHash, err := pathHashFromReq(r, allocationID) if err != nil { return nil, err } fileref, err := reference.GetReferenceFromLookupHash(ctx, allocationID, pathHash) - if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } + var ( + isOwner = clientID == alloc.OwnerID + isRepairer = clientID == alloc.RepairerID + isCollaborator = reference.IsACollaborator(ctx, fileref.ID, clientID) + ) + + if isOwner || isCollaborator{ + publicKey := alloc.OwnerPublicKey + if isCollaborator { + publicKey = ctx.Value(constants.ContextKeyClientKey).(string) + } + + valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), publicKey) + if !valid || err != nil { + return nil, common.NewError("invalid_signature", "Invalid signature") + } + } + result := fileref.GetListingData(ctx) commitMetaTxns, err := reference.GetCommitMetaTxns(ctx, fileref.ID) @@ -175,13 +190,6 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in result["collaborators"] = collaborators - // authorize file access - var ( - isOwner = clientID == alloc.OwnerID - isRepairer = clientID == alloc.RepairerID - isCollaborator = reference.IsACollaborator(ctx, fileref.ID, clientID) - ) - if !isOwner && !isRepairer && !isCollaborator { var authTokenString = r.FormValue("auth_token") @@ -189,7 +197,7 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in if isAuthorized, err := fsh.verifyAuthTicket(ctx, authTokenString, alloc, fileref, clientID, ); !isAuthorized { - return nil, common.NewErrorf("download_file", + return nil, common.NewErrorf("file_meta", "cannot verify auth ticket: %v", err) } @@ -376,7 +384,6 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i } fileref, err := reference.GetReferenceFromLookupHash(ctx, allocationID, pathHash) - if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } @@ -386,13 +393,13 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i } result := fileref.GetListingData(ctx) - stats, _ := stats.GetFileStats(ctx, fileref.ID) + fileStats, _ := stats.GetFileStats(ctx, fileref.ID) wm, _ := writemarker.GetWriteMarkerEntity(ctx, fileref.WriteMarker) - if wm != nil && stats != nil { - stats.WriteMarkerRedeemTxn = wm.CloseTxnID + if wm != nil && fileStats != nil { + fileStats.WriteMarkerRedeemTxn = wm.CloseTxnID } var statsMap map[string]interface{} - statsBytes, _ := json.Marshal(stats) + statsBytes, _ := json.Marshal(fileStats) if err = json.Unmarshal(statsBytes, &statsMap); err != nil { return nil, err } diff --git a/go.mod b/go.mod index a254994d0..e4e679d82 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.3.0-beta.3.0.20211028024808-1a15c1339eb2 + github.com/0chain/gosdk v1.3.0-beta.4.0.20211109103416-4d382fe5d5d4 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect @@ -25,7 +25,7 @@ require ( go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 + google.golang.org/genproto v0.0.0-20211101144312-62acf1d99145 google.golang.org/grpc v1.42.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index 494115773..081105fd0 100644 --- a/go.sum +++ b/go.sum @@ -34,7 +34,6 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -47,1485 +46,37 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/0chain/errors v1.0.2/go.mod h1:5t76jLb56TKfg/K2VD+eUMmNZJ42QsIRI8KzWuztwU4= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.3.0-beta.2 h1:+exlUvGFp1iG2UF+3Hken4CEfoYdHsBav+70be096IY= -github.com/0chain/gosdk v1.3.0-beta.2/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= -github.com/0chain/gosdk v1.3.0-beta.2.0.20211028024402-f93ca468ead1 h1:1BretsR6oIFEETv9LxiEnoSVT01BBYtqPSUam6NCzJI= -github.com/0chain/gosdk v1.3.0-beta.2.0.20211028024402-f93ca468ead1/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= -github.com/0chain/gosdk v1.3.0-beta.3.0.20211028024808-1a15c1339eb2 h1:PXk/ufOXPC7sUZaMvjOW70wsjJ/phtx4SwDMqqetarU= -github.com/0chain/gosdk v1.3.0-beta.3.0.20211028024808-1a15c1339eb2/go.mod h1:JtvcqYYWRdOVFm0pvjdKO5pCiItc/Is2f5wTuuA8F4M= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/0chain/gosdk v1.3.0-beta.4.0.20211109103416-4d382fe5d5d4 h1:p5EYDEE4GlAsS01nVwi+pqIU6afoiYqgQHTvRm+US1Y= +github.com/0chain/gosdk v1.3.0-beta.4.0.20211109103416-4d382fe5d5d4/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= -github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= -github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= -github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= -github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= -github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= -github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= -github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= -github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= -github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= -github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= -github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= -github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg= -github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= -github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= -github.com/dgraph-io/badger/v3 v3.2103.1/go.mod h1:dULbq6ehJ5K0cGW/1TQ9iSfUk0gbSiToDWmWmTsJ53E= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M= -github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY= -github.com/didip/tollbooth/v6 v6.1.1 h1:Nt7PvWLa9Y94OrykXsFNBinVRQIu8xdy4avpl99Dc1M= github.com/didip/tollbooth/v6 v6.1.1/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s= -github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.10.8 h1:0UP5WUR8hh46ffbjJV7PK499+uGEyasRIfffS0vy06o= -github.com/ethereum/go-ethereum v1.10.8/go.mod h1:pJNuIUYfX5+JKzSD/BTdNsvJSZ1TJqmz0dVyXMAbf6M= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= -github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= -github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.55.0 h1:0wVcG9udk2C3TGgmdIGKK9ScOZHZB5nbG+gwji9fhhc= github.com/go-ini/ini v1.55.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc= -github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 h1:rgxjzoDmDXw5q8HONgyHhBas4to0/XWRo/gPpJhsUNQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0/go.mod h1:qrJPVzv9YlhsrxJc3P/Q85nr0w1lIRikTl4JlhdDH5w= -github.com/h2non/filetype v1.0.9 h1:Y9YFg/WJNd7XoC5h3WD+GZSxHmuRRDyJQ7fcIlIJplI= -github.com/h2non/filetype v1.0.9/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= -github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 h1:TV+x87PxW2BcEzpX6yWK0mBEGc9NFppsihFszD6QmyA= github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= -github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= -github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= -github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= -github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= -github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= -github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= -github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= -github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= -github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= -github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= -github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= -github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= -github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= -github.com/jackc/pgconn v1.6.1/go.mod h1:g8mKMqmSUO6AzAvha7vy07g1rbGOlc7iF0nU0ei83hc= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= -github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= -github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= -github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= -github.com/jackc/pgtype v1.4.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= -github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= -github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= -github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= -github.com/jackc/pgx/v4 v4.7.1/go.mod h1:nu42q3aPjuC1M0Nak4bnoprKlXPINqopEKqbq5AZSC4= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= -github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= -github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= -github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= -github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/reedsolomon v1.9.11 h1:n2kipJFo+CPqg7fH988XJXjqEyj14RJ8BYj7UayxPNg= -github.com/klauspost/reedsolomon v1.9.11/go.mod h1:nLvuzNvy1ZDNQW30IuMc2ZWCbiqrJgdLoUS2X8HAUVg= -github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 h1:R7RAW1p8wjhlHKFhS4X7h8EePqADev/PltCmW9qlJoM= github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20/go.mod h1:sh5SGGmQVGUkWDnxevz0I2FJ4TeC18hRPRjKVBMb2kA= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= -github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magma/augmented-networks/accounting/protos v0.1.1/go.mod h1:Hpfg8aAxldUN7qlVtR5xwlAf8pcetFm8DWwRKZsh2J4= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= -github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= -github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= -github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= -github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= -github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= -github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= -github.com/selvatico/go-mocket v1.0.7 h1:jbVa7RkoOCzBanQYiYF+VWgySHZogg25fOIKkM38q5k= github.com/selvatico/go-mocket v1.0.7/go.mod h1:7bSWzuNieCdUlanCVu3w0ppS0LvDtPAZmKBIlhoTcp8= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= -github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= -github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= -github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= -github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= -github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= -go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= -go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= -go.dedis.ch/kyber/v3 v3.0.5 h1:BpjX6vY1R3b7TnJ0mUnCFRVXEJThSAj1zQzmNh4v+70= -go.dedis.ch/kyber/v3 v3.0.5/go.mod h1:V1z0JihG9+dUEUCKLI9j9tjnlIflBw3wx8UOg0g3Pnk= -go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRLo= -go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= -gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 h1:3V2dxSZpz4zozWWUq36vUxXEKnSYitEH2LdsAx+RUmg= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/genproto v0.0.0-20211101144312-62acf1d99145/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= -gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d h1:cEzkQplur9Z++gqjh48MF692Hkdl/jTkbo/7YQ5yssM= gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d/go.mod h1:n2DTgk9at7cr/CWOTKHWPaflj1fN+yuWpdK4lqSUbWA= -gorm.io/driver/mysql v0.3.1 h1:yvUT7Q0I3B9EHJ67NSp6cHbVwcdDHhVUsDAUiFFxRk0= -gorm.io/driver/mysql v0.3.1/go.mod h1:A7H1JD9dKdcjeUTpTuWKEC+E1a74qzW7/zaXqKaTbfM= -gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs= -gorm.io/driver/postgres v1.2.1 h1:JDQKnF7MC51dgL09Vbydc5kl83KkVDlcXfSPJ+xhh68= gorm.io/driver/postgres v1.2.1/go.mod h1:SHRZhu+D0tLOHV5qbxZRUM6kBcf3jp/kxPz2mYMTsNY= -gorm.io/driver/sqlite v1.0.8 h1:omllgSb7/eh9D6lGvLZOdU1ZElxdXuO3dn3Rk+dQxUE= -gorm.io/driver/sqlite v1.0.8/go.mod h1:xkm8/CEmA3yc4zRd0pdCqm43BjO8Hm6avfTpxWb/7c4= -gorm.io/driver/sqlserver v0.2.5 h1:o/MXpn9/BB68RXEEQzfhsSL382yEqUtdCiGIuCspmkY= -gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+odcueLA= -gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.22.0/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.2 h1:1iKcvyJnR5bHydBhDqTwasOkoo6+o4Ms5cknSt6qP7I= gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 66b6c210fb2dffa62287b24d42421571554728c3 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 10 Nov 2021 23:23:33 +0800 Subject: [PATCH 042/161] fix: rollback to herumi bls (#425) * refactor(main):improved startup code * fix(gomod): fixed go.mod * fix(gomod): rollback to herumi bls * fix(gomod): rollback to herumi bls * fix(test): fixed NewSignatureScheme * fix(test): fixed signatureScheme.RecoverKeys * fix(test): fixed chunk_size issue on tests * fix(test): fixed ipv6 issue on integration tests * fix(test): reverted uncomment for debugging * fix(devops): removed build validator from integration tests * fix(devops): split integrations-tests into smaller step * fix(devops): added new shell script and docker-compose for github action * fix(devops): fixed docker-compose-github.yml * fix(devops): run docker-compose as deamon * fix(devops): print docker ps for debugging * fix(devops): expost ports --- .github/workflows/ci.yml | 26 +- .gitignore | 2 +- code/go/0chain.net/blobber/config.go | 104 ++ code/go/0chain.net/blobber/datastore.go | 34 + code/go/0chain.net/blobber/filestore.go | 19 + code/go/0chain.net/blobber/flags.go | 56 + code/go/0chain.net/blobber/grpc.go | 9 +- code/go/0chain.net/blobber/http.go | 102 ++ code/go/0chain.net/blobber/logging.go | 23 + code/go/0chain.net/blobber/main.go | 482 +----- code/go/0chain.net/blobber/minio.go | 54 + code/go/0chain.net/blobber/node.go | 37 + code/go/0chain.net/blobber/worker.go | 45 + code/go/0chain.net/blobber/zcn.go | 115 ++ .../blobbercore/handler/handler_test.go | 20 +- .../handler/helper_integration_test.go | 2 +- .../0chain.net/core/encryption/keys_test.go | 24 +- docker.local/b0docker-compose-github.yml | 64 + docker.local/bin/blobber.start_github.sh | 9 + go.mod | 2 +- go.sum | 1468 ++++++++++++++++- 21 files changed, 2191 insertions(+), 506 deletions(-) create mode 100644 code/go/0chain.net/blobber/config.go create mode 100644 code/go/0chain.net/blobber/datastore.go create mode 100644 code/go/0chain.net/blobber/filestore.go create mode 100644 code/go/0chain.net/blobber/flags.go create mode 100644 code/go/0chain.net/blobber/http.go create mode 100644 code/go/0chain.net/blobber/logging.go create mode 100644 code/go/0chain.net/blobber/minio.go create mode 100644 code/go/0chain.net/blobber/node.go create mode 100644 code/go/0chain.net/blobber/worker.go create mode 100644 code/go/0chain.net/blobber/zcn.go create mode 100644 docker.local/b0docker-compose-github.yml create mode 100755 docker.local/bin/blobber.start_github.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c5090a16..fc79a8d72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,28 +21,32 @@ jobs: timeout-minutes: 40 steps: - - name: Setup go 1.16 + - name: Setup go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' # The Go version to download (if necessary) and use. + go-version: '1.17' # The Go version to download (if necessary) and use. - name: Clone Blobber uses: actions/checkout@v1 - - name: Build test environment and run tests + - name: Build Blobber run: | - sed -i '/#expose_ci_port/c\ ports:\n - "5432:5432"' ./docker.local/b0docker-compose.yml - ./docker.local/bin/blobber.init.setup.sh docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 ./docker.local/bin/build.base.sh ./docker.local/bin/build.blobber.sh - ./docker.local/bin/build.validator.sh + + - name: Run Blobber on gRPC + run: | + ./docker.local/bin/blobber.init.setup.sh cd docker.local/blobber1 - ../bin/blobber.start_bls.sh /dev/null & - cd ../.. - - go16=$(which go) - sudo $go16 test -tags bn256 ./... -args integration; + ../bin/blobber.start_github.sh + docker ps + + - name: Run Tests + run: | + go17=$(which go) + sudo $go17 test -tags bn256 ./... -args integration + golangci: name: lint runs-on: [self-hosted, tests-suite] diff --git a/.gitignore b/.gitignore index 7e5599120..3ba7a8d29 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ __debug_bin **/pkg/ dev.local/data out/ - +**/tmp/ diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go new file mode 100644 index 000000000..39d99aac2 --- /dev/null +++ b/code/go/0chain.net/blobber/config.go @@ -0,0 +1,104 @@ +package main + +import ( + "fmt" + "log" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "github.com/spf13/viper" +) + +func setupConfig() { + fmt.Print("[2/10] load config") + // setup default + config.SetupDefaultConfig() + + // setup config file + config.SetupConfig(configDir) + + // load config + config.Configuration.DeploymentMode = byte(deploymentMode) + config.Configuration.ChainID = viper.GetString("server_chain.id") + config.Configuration.SignatureScheme = viper.GetString("server_chain.signature_scheme") + + config.Configuration.ContentRefWorkerFreq = viper.GetInt64("contentref_cleaner.frequency") + config.Configuration.ContentRefWorkerTolerance = viper.GetInt64("contentref_cleaner.tolerance") + + config.Configuration.OpenConnectionWorkerFreq = viper.GetInt64("openconnection_cleaner.frequency") + config.Configuration.OpenConnectionWorkerTolerance = viper.GetInt64("openconnection_cleaner.tolerance") + + config.Configuration.WMRedeemFreq = viper.GetInt64("writemarker_redeem.frequency") + config.Configuration.WMRedeemNumWorkers = viper.GetInt("writemarker_redeem.num_workers") + + config.Configuration.RMRedeemFreq = viper.GetInt64("readmarker_redeem.frequency") + config.Configuration.RMRedeemNumWorkers = viper.GetInt("readmarker_redeem.num_workers") + + config.Configuration.ChallengeResolveFreq = viper.GetInt64("challenge_response.frequency") + config.Configuration.ChallengeResolveNumWorkers = viper.GetInt("challenge_response.num_workers") + config.Configuration.ChallengeMaxRetires = viper.GetInt("challenge_response.max_retries") + + config.Configuration.ColdStorageMinimumFileSize = viper.GetInt64("cold_storage.min_file_size") + config.Configuration.ColdStorageTimeLimitInHours = viper.GetInt64("cold_storage.file_time_limit_in_hours") + config.Configuration.ColdStorageJobQueryLimit = viper.GetInt64("cold_storage.job_query_limit") + config.Configuration.ColdStorageStartCapacitySize = viper.GetInt64("cold_storage.start_capacity_size") + config.Configuration.ColdStorageDeleteLocalCopy = viper.GetBool("cold_storage.delete_local_copy") + config.Configuration.ColdStorageDeleteCloudCopy = viper.GetBool("cold_storage.delete_cloud_copy") + + config.Configuration.MinioStart = viper.GetBool("minio.start") + config.Configuration.MinioWorkerFreq = viper.GetInt64("minio.worker_frequency") + config.Configuration.MinioUseSSL = viper.GetBool("minio.use_ssl") + + config.Configuration.Capacity = viper.GetInt64("capacity") + config.Configuration.MaxFileSize = viper.GetInt64("max_file_size") + + config.Configuration.DBHost = viper.GetString("db.host") + config.Configuration.DBName = viper.GetString("db.name") + config.Configuration.DBPort = viper.GetString("db.port") + config.Configuration.DBUserName = viper.GetString("db.user") + config.Configuration.DBPassword = viper.GetString("db.password") + + config.Configuration.Capacity = viper.GetInt64("capacity") + config.Configuration.ReadPrice = viper.GetFloat64("read_price") + config.Configuration.WritePrice = viper.GetFloat64("write_price") + config.Configuration.PriceInUSD = viper.GetBool("price_in_usd") + config.Configuration.MinLockDemand = viper.GetFloat64("min_lock_demand") + config.Configuration.MaxOfferDuration = viper.GetDuration("max_offer_duration") + config.Configuration.ChallengeCompletionTime = viper.GetDuration("challenge_completion_time") + + config.Configuration.ReadLockTimeout = int64( + viper.GetDuration("read_lock_timeout") / time.Second, + ) + config.Configuration.WriteLockTimeout = int64( + viper.GetDuration("write_lock_timeout") / time.Second, + ) + + config.Configuration.UpdateAllocationsInterval = + viper.GetDuration("update_allocations_interval") + + config.Configuration.DelegateWallet = viper.GetString("delegate_wallet") + if w := config.Configuration.DelegateWallet; len(w) != 64 { + log.Fatal("invalid delegate wallet:", w) + } + config.Configuration.MinStake = int64(viper.GetFloat64("min_stake") * 1e10) + config.Configuration.MaxStake = int64(viper.GetFloat64("max_stake") * 1e10) + config.Configuration.NumDelegates = viper.GetInt("num_delegates") + config.Configuration.ServiceCharge = viper.GetFloat64("service_charge") + + config.Configuration.MinSubmit = viper.GetInt("min_submit") + if config.Configuration.MinSubmit < 1 { + config.Configuration.MinSubmit = 50 + } else if config.Configuration.MinSubmit > 100 { + config.Configuration.MinSubmit = 100 + } + config.Configuration.MinConfirmation = viper.GetInt("min_confirmation") + if config.Configuration.MinConfirmation < 1 { + config.Configuration.MinConfirmation = 50 + } else if config.Configuration.MinConfirmation > 100 { + config.Configuration.MinConfirmation = 100 + } + + transaction.MinConfirmation = config.Configuration.MinConfirmation + fmt.Print(" [OK]\n") +} diff --git a/code/go/0chain.net/blobber/datastore.go b/code/go/0chain.net/blobber/datastore.go new file mode 100644 index 000000000..bc140634f --- /dev/null +++ b/code/go/0chain.net/blobber/datastore.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" +) + +func setupDatabase() error { + fmt.Print("\r[7/10] connect data store") + // check for database connection + for i := 0; i < 600; i++ { + + if i > 0 { + fmt.Printf("\r[7/10] connect(%v) data store", i) + } + + if err := datastore.GetStore().Open(); err == nil { + if i == 1 { // no more attempts + logging.Logger.Error("Failed to connect to the database. Shutting the server down") + return err + } + fmt.Print(" [OK]\n") + return nil + } + + time.Sleep(1 * time.Second) + + } + + return nil +} diff --git a/code/go/0chain.net/blobber/filestore.go b/code/go/0chain.net/blobber/filestore.go new file mode 100644 index 000000000..1c7577e09 --- /dev/null +++ b/code/go/0chain.net/blobber/filestore.go @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" +) + +var fsStore filestore.FileStore //nolint:unused // global which might be needed somewhere + +func setupFileStore() (err error) { + fmt.Print("[8/10] setup file store") + + fsStore, err = filestore.SetupFSStore(filesDir + "/files") + + fmt.Print(" [OK]\n") + + return err +} diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go new file mode 100644 index 000000000..d00a79158 --- /dev/null +++ b/code/go/0chain.net/blobber/flags.go @@ -0,0 +1,56 @@ +package main + +import ( + "flag" + "fmt" +) + +var ( + deploymentMode int + keysFile string + minioFile string + filesDir string + metadataDB string + logDir string + httpPort int + hostname string + configDir string + grpcPort int +) + +func init() { + + flag.IntVar(&deploymentMode, "deployment_mode", 2, "deployment mode: 0=dev,1=test, 2=mainnet") + flag.StringVar(&keysFile, "keys_file", "", "keys_file") + flag.StringVar(&minioFile, "minio_file", "", "minio_file") + flag.StringVar(&filesDir, "files_dir", "", "files_dir") + flag.StringVar(&metadataDB, "db_dir", "", "db_dir") + flag.StringVar(&logDir, "log_dir", "", "log_dir") + flag.IntVar(&httpPort, "port", 0, "port") + flag.StringVar(&hostname, "hostname", "", "hostname") + flag.StringVar(&configDir, "config_dir", "./config", "config_dir") + + flag.IntVar(&grpcPort, "grpc_port", 0, "grpc_port") +} + +func parseFlags() { + fmt.Print("[1/10] load flags") + flag.Parse() + + if filesDir == "" { + panic("Please specify --files_dir absolute folder name option where uploaded files can be stored") + } + + if metadataDB == "" { + panic("Please specify --db_dir absolute folder name option where meta data db can be stored") + } + + if hostname == "" { + panic("Please specify --hostname which is the public hostname") + } + + if httpPort <= 0 { + panic("Please specify --port which is the port on which requests are accepted") + } + fmt.Print(" [OK]\n") +} diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go index 298ad043e..04d0462a2 100644 --- a/code/go/0chain.net/blobber/grpc.go +++ b/code/go/0chain.net/blobber/grpc.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "net" + "strconv" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" "github.com/0chain/blobber/code/go/0chain.net/core/logging" @@ -13,18 +14,18 @@ import ( "google.golang.org/grpc/reflection" ) -func startGRPCServer(r mux.Router, port string) { +func startGRPCServer(r mux.Router) { grpcServer := handler.NewGRPCServerWithMiddlewares(&r) reflection.Register(grpcServer) - if port == "" { + if grpcPort <= 0 { logging.Logger.Error("Could not start grpc server since grpc port has not been specified." + " Please specify the grpc port in the --grpc_port build arguement to start the grpc server") return } - logging.Logger.Info("listening too grpc requests on port - " + port) - lis, err := net.Listen("tcp", fmt.Sprintf(":%s", port)) + logging.Logger.Info("listening too grpc requests on port - " + strconv.Itoa(grpcPort)) + lis, err := net.Listen("tcp", fmt.Sprintf(":%v", grpcPort)) if err != nil { log.Fatalf("failed to listen: %v", err) } diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go new file mode 100644 index 000000000..aed7d72bc --- /dev/null +++ b/code/go/0chain.net/blobber/http.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "runtime" + "strconv" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" + "github.com/0chain/blobber/code/go/0chain.net/core/build" + "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/gosdk/zcncore" + "github.com/gorilla/mux" + "go.uber.org/zap" +) + +var startTime time.Time + +func startHttpServer() { + fmt.Println("[10/10] start http server [OK]") + + mode := "main net" + if config.Development() { + mode = "development" + } else if config.TestNet() { + mode = "test net" + } + + logging.Logger.Info("Starting blobber", zap.Int("available_cpus", runtime.NumCPU()), zap.Int("port", httpPort), zap.String("chain_id", config.GetServerChainID()), zap.String("mode", mode)) + + //address := publicIP + ":" + portString + address := ":" + strconv.Itoa(httpPort) + var server *http.Server + + r := mux.NewRouter() + + common.ConfigRateLimits() + initHandlers(r) + + if config.Development() { + // No WriteTimeout setup to enable pprof + server = &http.Server{ + Addr: address, + ReadHeaderTimeout: 30 * time.Second, + MaxHeaderBytes: 1 << 20, + Handler: r, + } + } else { + server = &http.Server{ + Addr: address, + ReadHeaderTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 30 * time.Second, + MaxHeaderBytes: 1 << 20, + Handler: r, + } + } + common.HandleShutdown(server) + handler.HandleShutdown(common.GetRootContext()) + + logging.Logger.Info("Ready to listen to the requests") + + if config.Development() { + go startGRPCServer(*r) + } + + startTime = time.Now().UTC() + + log.Fatal(server.ListenAndServe()) +} + +func initHandlers(r *mux.Router) { + r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + mc := chain.GetServerChain() + + fmt.Fprintf(w, "
Running since %v ...\n", startTime) + fmt.Fprintf(w, "
Working on the chain: %v
\n", mc.ID) + fmt.Fprintf(w, + "
I am a blobber with
  • id:%v
  • public_key:%v
  • build_tag:%v
\n", + node.Self.ID, node.Self.PublicKey, build.BuildTag, + ) + + fmt.Fprintf(w, "
Miners ...\n") + network := zcncore.GetNetwork() + for _, miner := range network.Miners { + fmt.Fprintf(w, "%v\n", miner) + } + + fmt.Fprintf(w, "
Sharders ...\n") + for _, sharder := range network.Sharders { + fmt.Fprintf(w, "%v\n", sharder) + } + }) + + handler.SetupHandlers(r) +} diff --git a/code/go/0chain.net/blobber/logging.go b/code/go/0chain.net/blobber/logging.go new file mode 100644 index 000000000..dde22742c --- /dev/null +++ b/code/go/0chain.net/blobber/logging.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/zcncore" +) + +func setupLogging() { + fmt.Print("[3/10] init logging") + + if config.Development() { + logging.InitLogging("development", logDir, "0chainBlobber.log") + } else { + logging.InitLogging("production", logDir, "0chainBlobber.log") + } + + zcncore.SetLogFile(logDir+"/0chainBlobber.log", false) + zcncore.SetLogLevel(3) + fmt.Print(" [OK]\n") +} diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 08927f196..06fb1f8d6 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -1,491 +1,47 @@ package main import ( - "bufio" - "flag" - "fmt" - "io" - "log" - "net/http" - "os" - "runtime" - "strconv" - "time" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/challenge" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" - "github.com/0chain/blobber/code/go/0chain.net/core/build" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - - "github.com/0chain/gosdk/zcncore" - "github.com/gorilla/mux" - "github.com/spf13/viper" - "go.uber.org/zap" ) -var startTime time.Time -var serverChain *chain.Chain -var filesDir *string -var metadataDB *string - -func initHandlers(r *mux.Router) { - r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - mc := chain.GetServerChain() - - fmt.Fprintf(w, "
Running since %v ...\n", startTime) - fmt.Fprintf(w, "
Working on the chain: %v
\n", mc.ID) - fmt.Fprintf(w, - "
I am a blobber with
  • id:%v
  • public_key:%v
  • build_tag:%v
\n", - node.Self.ID, node.Self.PublicKey, build.BuildTag, - ) - - fmt.Fprintf(w, "
Miners ...\n") - network := zcncore.GetNetwork() - for _, miner := range network.Miners { - fmt.Fprintf(w, "%v\n", miner) - } - - fmt.Fprintf(w, "
Sharders ...\n") - for _, sharder := range network.Sharders { - fmt.Fprintf(w, "%v\n", sharder) - } - }) - - handler.SetupHandlers(r) -} - -var fsStore filestore.FileStore //nolint:unused // global which might be needed somewhere - -func initEntities() (err error) { - fsStore, err = filestore.SetupFSStore(*filesDir + "/files") - return err -} - -func setupWorkerConfig() { - config.Configuration.ContentRefWorkerFreq = viper.GetInt64("contentref_cleaner.frequency") - config.Configuration.ContentRefWorkerTolerance = viper.GetInt64("contentref_cleaner.tolerance") - - config.Configuration.OpenConnectionWorkerFreq = viper.GetInt64("openconnection_cleaner.frequency") - config.Configuration.OpenConnectionWorkerTolerance = viper.GetInt64("openconnection_cleaner.tolerance") - - config.Configuration.WMRedeemFreq = viper.GetInt64("writemarker_redeem.frequency") - config.Configuration.WMRedeemNumWorkers = viper.GetInt("writemarker_redeem.num_workers") - - config.Configuration.RMRedeemFreq = viper.GetInt64("readmarker_redeem.frequency") - config.Configuration.RMRedeemNumWorkers = viper.GetInt("readmarker_redeem.num_workers") - - config.Configuration.ChallengeResolveFreq = viper.GetInt64("challenge_response.frequency") - config.Configuration.ChallengeResolveNumWorkers = viper.GetInt("challenge_response.num_workers") - config.Configuration.ChallengeMaxRetires = viper.GetInt("challenge_response.max_retries") - - config.Configuration.ColdStorageMinimumFileSize = viper.GetInt64("cold_storage.min_file_size") - config.Configuration.ColdStorageTimeLimitInHours = viper.GetInt64("cold_storage.file_time_limit_in_hours") - config.Configuration.ColdStorageJobQueryLimit = viper.GetInt64("cold_storage.job_query_limit") - config.Configuration.ColdStorageStartCapacitySize = viper.GetInt64("cold_storage.start_capacity_size") - config.Configuration.ColdStorageDeleteLocalCopy = viper.GetBool("cold_storage.delete_local_copy") - config.Configuration.ColdStorageDeleteCloudCopy = viper.GetBool("cold_storage.delete_cloud_copy") - - config.Configuration.MinioStart = viper.GetBool("minio.start") - config.Configuration.MinioWorkerFreq = viper.GetInt64("minio.worker_frequency") - config.Configuration.MinioUseSSL = viper.GetBool("minio.use_ssl") - - config.Configuration.Capacity = viper.GetInt64("capacity") - config.Configuration.MaxFileSize = viper.GetInt64("max_file_size") - - config.Configuration.DBHost = viper.GetString("db.host") - config.Configuration.DBName = viper.GetString("db.name") - config.Configuration.DBPort = viper.GetString("db.port") - config.Configuration.DBUserName = viper.GetString("db.user") - config.Configuration.DBPassword = viper.GetString("db.password") - - config.Configuration.Capacity = viper.GetInt64("capacity") - config.Configuration.ReadPrice = viper.GetFloat64("read_price") - config.Configuration.WritePrice = viper.GetFloat64("write_price") - config.Configuration.PriceInUSD = viper.GetBool("price_in_usd") - config.Configuration.MinLockDemand = viper.GetFloat64("min_lock_demand") - config.Configuration.MaxOfferDuration = viper.GetDuration("max_offer_duration") - config.Configuration.ChallengeCompletionTime = viper.GetDuration("challenge_completion_time") - - config.Configuration.ReadLockTimeout = int64( - viper.GetDuration("read_lock_timeout") / time.Second, - ) - config.Configuration.WriteLockTimeout = int64( - viper.GetDuration("write_lock_timeout") / time.Second, - ) - - config.Configuration.UpdateAllocationsInterval = - viper.GetDuration("update_allocations_interval") - - config.Configuration.DelegateWallet = viper.GetString("delegate_wallet") - if w := config.Configuration.DelegateWallet; len(w) != 64 { - log.Fatal("invalid delegate wallet:", w) - } - config.Configuration.MinStake = int64(viper.GetFloat64("min_stake") * 1e10) - config.Configuration.MaxStake = int64(viper.GetFloat64("max_stake") * 1e10) - config.Configuration.NumDelegates = viper.GetInt("num_delegates") - config.Configuration.ServiceCharge = viper.GetFloat64("service_charge") - - config.Configuration.MinSubmit = viper.GetInt("min_submit") - if config.Configuration.MinSubmit < 1 { - config.Configuration.MinSubmit = 50 - } else if config.Configuration.MinSubmit > 100 { - config.Configuration.MinSubmit = 100 - } - config.Configuration.MinConfirmation = viper.GetInt("min_confirmation") - if config.Configuration.MinConfirmation < 1 { - config.Configuration.MinConfirmation = 50 - } else if config.Configuration.MinConfirmation > 100 { - config.Configuration.MinConfirmation = 100 - } - - transaction.MinConfirmation = config.Configuration.MinConfirmation -} - -func setupMinioConfig(reader io.Reader) error { - scanner := bufio.NewScanner(reader) - more := scanner.Scan() - if !more { - return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") - } - - filestore.MinioConfig.StorageServiceURL = scanner.Text() - more = scanner.Scan() - if !more { - return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") - } - - filestore.MinioConfig.AccessKeyID = scanner.Text() - more = scanner.Scan() - if !more { - return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") - } - - filestore.MinioConfig.SecretAccessKey = scanner.Text() - more = scanner.Scan() - if !more { - return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") - } - - filestore.MinioConfig.BucketName = scanner.Text() - more = scanner.Scan() - if !more { - return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") - } - - filestore.MinioConfig.BucketLocation = scanner.Text() - return nil -} - -func setupWorkers() { - var root = common.GetRootContext() - handler.SetupWorkers(root) - challenge.SetupWorkers(root) - readmarker.SetupWorkers(root) - writemarker.SetupWorkers(root) - allocation.StartUpdateWorker(root, - config.Configuration.UpdateAllocationsInterval) -} - -func setupDatabase() { - // check for database connection - for i := 600; i > 0; i-- { - time.Sleep(1 * time.Second) - if err := datastore.GetStore().Open(); err == nil { - if i == 1 { // no more attempts - Logger.Error("Failed to connect to the database. Shutting the server down") - panic(err) // fail - } - - return // success - } - } -} - -func setupOnChain() { - const ATTEMPT_DELAY = 60 * 1 // 1 minute - - // setup wallet - if err := handler.WalletRegister(); err != nil { - panic(err) - } - - // setup blobber (add or update) on the blockchain (multiple attempts) - for i := 10; i > 0; i-- { - if err := addOrUpdateOnChain(); err != nil { - if i == 1 { // no more attempts - panic(err) - } - } else { - break - } - - time.Sleep(ATTEMPT_DELAY * time.Second) - } - - setupWorkers() - - go healthCheckOnChainWorker() - - if config.Configuration.PriceInUSD { - go addOrUpdateOnChainWorker() - } -} - -func addOrUpdateOnChain() error { - txnHash, err := handler.BlobberAdd(common.GetRootContext()) - if err != nil { - return err - } - - if t, err := handler.TransactionVerify(txnHash); err != nil { - Logger.Error("Failed to verify blobber add/update transaction", zap.Any("err", err), zap.String("txn.Hash", txnHash)) - } else { - Logger.Info("Verified blobber add/update transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) - } - - return err -} - -func addOrUpdateOnChainWorker() { - var REPEAT_DELAY = 60 * 60 * time.Duration(viper.GetInt("price_worker_in_hours")) // 12 hours with default settings - for { - time.Sleep(REPEAT_DELAY * time.Second) - if err := addOrUpdateOnChain(); err != nil { - continue // pass // required by linting - } - } -} - -func healthCheckOnChain() error { - txnHash, err := handler.BlobberHealthCheck(common.GetRootContext()) - if err != nil { - if err == handler.ErrBlobberHasRemoved { - return nil - } else { - return err - } - } - - if t, err := handler.TransactionVerify(txnHash); err != nil { - Logger.Error("Failed to verify blobber health check", zap.Any("err", err), zap.String("txn.Hash", txnHash)) - } else { - Logger.Info("Verified blobber health check", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) - } - - return err -} - -func healthCheckOnChainWorker() { - const REPEAT_DELAY = 60 * 15 // 15 minutes - - for { - time.Sleep(REPEAT_DELAY * time.Second) - if err := healthCheckOnChain(); err != nil { - continue // pass // required by linting - } - } -} - -func setup(logDir string) error { - // init blockchain related stuff - zcncore.SetLogFile(logDir+"/0chainBlobber.log", false) - zcncore.SetLogLevel(3) - if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil { - return err - } - if err := zcncore.SetWalletInfo(node.Self.GetWalletString(), false); err != nil { - return err - } - - // setup on blockchain - go setupOnChain() - return nil -} - -// // Comment out to pass lint. Still keep this function around in case we want to -// // change how CORS validates origins. -// func isValidOrigin(origin string) bool { -// var url, err = url.Parse(origin) -// if err != nil { -// return false -// } -// var host = url.Hostname() -// if host == "localhost" { -// return true -// } -// if host == "0chain.net" || host == "0box.io" || -// strings.HasSuffix(host, ".0chain.net") || -// strings.HasSuffix(host, ".alphanet-0chain.net") || -// strings.HasSuffix(host, ".testnet-0chain.net") || -// strings.HasSuffix(host, ".devnet-0chain.net") || -// strings.HasSuffix(host, ".mainnet-0chain.net") { -// return true -// } -// return false -// } - func main() { + parseFlags() - grpcPortString := "" - - deploymentMode := flag.Int("deployment_mode", 2, "deployment_mode") - keysFile := flag.String("keys_file", "", "keys_file") - minioFile := flag.String("minio_file", "", "minio_file") - filesDir = flag.String("files_dir", "", "files_dir") - metadataDB = flag.String("db_dir", "", "db_dir") - logDir := flag.String("log_dir", "", "log_dir") - portString := flag.String("port", "", "port") - hostname := flag.String("hostname", "", "hostname") - configDir := flag.String("config_dir", "./config", "config_dir") + setupConfig() - flag.StringVar(&grpcPortString, "grpc_port", "", "grpc_port") - - flag.Parse() - - config.SetupDefaultConfig() - - config.SetupConfig(*configDir) - - config.Configuration.DeploymentMode = byte(*deploymentMode) - - if config.Development() { - logging.InitLogging("development", *logDir, "0chainBlobber.log") - } else { - logging.InitLogging("production", *logDir, "0chainBlobber.log") - } - config.Configuration.ChainID = viper.GetString("server_chain.id") - config.Configuration.SignatureScheme = viper.GetString("server_chain.signature_scheme") - setupWorkerConfig() - - if *filesDir == "" { - panic("Please specify --files_dir absolute folder name option where uploaded files can be stored") - } + setupLogging() - if *metadataDB == "" { - panic("Please specify --db_dir absolute folder name option where meta data db can be stored") - } - - if *hostname == "" { - panic("Please specify --hostname which is the public hostname") - } - - if *portString == "" { - panic("Please specify --port which is the port on which requests are accepted") - } - - reader, err := os.Open(*keysFile) - if err != nil { + if err := setupMinio(); err != nil { + logging.Logger.Error("Error setting up minio " + err.Error()) panic(err) } - publicKey, privateKey, _, _ := encryption.ReadKeys(reader) - reader.Close() - - reader, err = os.Open(*minioFile) - if err != nil { + if err := setupNode(); err != nil { + logging.Logger.Error("Error setting up blobber node " + err.Error()) panic(err) } - err = setupMinioConfig(reader) - if err != nil { + if err := setupServerChain(); err != nil { + logging.Logger.Error("Error setting up server chain" + err.Error()) panic(err) } - reader.Close() - - node.Self.SetKeys(publicKey, privateKey) - - port, err := strconv.Atoi(*portString) //fmt.Sprintf(":%v", port) // node.Self.Port - if err != nil { - Logger.Panic("Port specified is not Int " + *portString) - return - } - - node.Self.SetHostURL(*hostname, port) - Logger.Info(" Base URL" + node.Self.GetURLBase()) - - config.SetServerChainID(config.Configuration.ChainID) - common.SetupRootContext(node.GetNodeContext()) - //ctx := common.GetRootContext() - serverChain = chain.NewChainFromConfig() - - if node.Self.ID == "" { - Logger.Panic("node definition for self node doesn't exist") - } else { - Logger.Info("self identity", zap.Any("id", node.Self.ID)) + if err := setupDatabase(); err != nil { + logging.Logger.Error("Error setting up data store" + err.Error()) + panic(err) } - initIntegrationsTests(node.Self.ID) - - //address := publicIP + ":" + portString - address := ":" + *portString - - chain.SetServerChain(serverChain) - - setupDatabase() - // Initialize after server chain is setup. - if err := initEntities(); err != nil { - Logger.Error("Error setting up blobber on blockchian" + err.Error()) - } - if err := setup(*logDir); err != nil { - Logger.Error("Error setting up blobber on blockchian" + err.Error()) - } - mode := "main net" - if config.Development() { - mode = "development" - } else if config.TestNet() { - mode = "test net" - } - Logger.Info("Starting blobber", zap.Int("available_cpus", runtime.NumCPU()), zap.String("port", *portString), zap.String("chain_id", config.GetServerChainID()), zap.String("mode", mode)) - - var server *http.Server - - r := mux.NewRouter() - - common.ConfigRateLimits() - initHandlers(r) - - if config.Development() { - // No WriteTimeout setup to enable pprof - server = &http.Server{ - Addr: address, - ReadHeaderTimeout: 30 * time.Second, - MaxHeaderBytes: 1 << 20, - Handler: r, - } - } else { - server = &http.Server{ - Addr: address, - ReadHeaderTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 30 * time.Second, - MaxHeaderBytes: 1 << 20, - Handler: r, - } + if err := setupFileStore(); err != nil { + logging.Logger.Error("Error setting up file store" + err.Error()) + panic(err) } - common.HandleShutdown(server) - handler.HandleShutdown(common.GetRootContext()) - - Logger.Info("Ready to listen to the requests") - if config.Development() { - go startGRPCServer(*r, grpcPortString) - } + // only enabled with "// +build integration_tests" + initIntegrationsTests(node.Self.ID) - startTime = time.Now().UTC() + go setupOnChain() - log.Fatal(server.ListenAndServe()) + startHttpServer() } diff --git a/code/go/0chain.net/blobber/minio.go b/code/go/0chain.net/blobber/minio.go new file mode 100644 index 000000000..7b4417448 --- /dev/null +++ b/code/go/0chain.net/blobber/minio.go @@ -0,0 +1,54 @@ +package main + +import ( + "bufio" + "fmt" + "os" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" +) + +func setupMinio() error { + fmt.Print("[4/10] setup minio") + reader, err := os.Open(minioFile) + if err != nil { + return err + } + defer reader.Close() + + scanner := bufio.NewScanner(reader) + more := scanner.Scan() + if !more { + return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") + } + + filestore.MinioConfig.StorageServiceURL = scanner.Text() + more = scanner.Scan() + if !more { + return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") + } + + filestore.MinioConfig.AccessKeyID = scanner.Text() + more = scanner.Scan() + if !more { + return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") + } + + filestore.MinioConfig.SecretAccessKey = scanner.Text() + more = scanner.Scan() + if !more { + return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") + } + + filestore.MinioConfig.BucketName = scanner.Text() + more = scanner.Scan() + if !more { + return common.NewError("process_minio_config_failed", "Unable to read minio config from minio config file") + } + + filestore.MinioConfig.BucketLocation = scanner.Text() + + fmt.Print(" [OK]\n") + return nil +} diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go new file mode 100644 index 000000000..710a68ed9 --- /dev/null +++ b/code/go/0chain.net/blobber/node.go @@ -0,0 +1,37 @@ +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/node" + "go.uber.org/zap" +) + +func setupNode() error { + fmt.Print("[5/10] setup blobber") + + reader, err := os.Open(keysFile) + if err != nil { + + return err + } + defer reader.Close() + + publicKey, privateKey, _, _ := encryption.ReadKeys(reader) + + node.Self.SetKeys(publicKey, privateKey) + if node.Self.ID == "" { + return errors.New("node definition for self node doesn't exist") + } else { + logging.Logger.Info("self identity", zap.Any("id", node.Self.ID)) + } + + node.Self.SetHostURL(hostname, httpPort) + logging.Logger.Info(" Base URL" + node.Self.GetURLBase()) + fmt.Print(" [OK]\n") + return nil +} diff --git a/code/go/0chain.net/blobber/worker.go b/code/go/0chain.net/blobber/worker.go new file mode 100644 index 000000000..0a0cef16b --- /dev/null +++ b/code/go/0chain.net/blobber/worker.go @@ -0,0 +1,45 @@ +package main + +import ( + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/challenge" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/spf13/viper" +) + +func setupWorkers() { + var root = common.GetRootContext() + handler.SetupWorkers(root) + challenge.SetupWorkers(root) + readmarker.SetupWorkers(root) + writemarker.SetupWorkers(root) + allocation.StartUpdateWorker(root, + config.Configuration.UpdateAllocationsInterval) +} + +func refreshPriceOnChain() { + var REPEAT_DELAY = 60 * 60 * time.Duration(viper.GetInt("price_worker_in_hours")) // 12 hours with default settings + for { + time.Sleep(REPEAT_DELAY * time.Second) + if err := registerBlobberOnChain(); err != nil { + continue // pass // required by linting + } + } +} + +func keepAliveOnChain() { + const REPEAT_DELAY = 60 * 15 // 15 minutes + + for { + time.Sleep(REPEAT_DELAY * time.Second) + if err := healthCheckOnChain(); err != nil { + continue // pass // required by linting + } + } +} diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go new file mode 100644 index 000000000..4023dff1f --- /dev/null +++ b/code/go/0chain.net/blobber/zcn.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" + "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/gosdk/zcncore" + "go.uber.org/zap" +) + +func setupOnChain() { + fmt.Println("[9/10] connecting to chain ") + + const ATTEMPT_DELAY = 60 * 1 + + // setup wallet + fmt.Print(" + connect to miners: ") + if err := handler.WalletRegister(); err != nil { + fmt.Println(err.Error() + "\n") + panic(err) + } + fmt.Print(" [OK]\n") + + // setup blobber (add or update) on the blockchain (multiple attempts) + for i := 1; i <= 10; i++ { + if i == 1 { + fmt.Printf("\r + connect to sharders:") + } else { + fmt.Printf("\r + [%v/10]connect to sharders:", i) + } + + if err := registerBlobberOnChain(); err != nil { + if i == 10 { // no more attempts + panic(err) + } + fmt.Print("\n ", err.Error()+"\n") + + } else { + fmt.Print(" [OK]\n") + break + } + for n := 0; n < ATTEMPT_DELAY; n++ { + <-time.After(1 * time.Second) + + fmt.Printf("\r - wait %v seconds to retry", ATTEMPT_DELAY-n) + } + + } + + go setupWorkers() + + go keepAliveOnChain() + + if config.Configuration.PriceInUSD { + go refreshPriceOnChain() + } +} + +func registerBlobberOnChain() error { + txnHash, err := handler.BlobberAdd(common.GetRootContext()) + if err != nil { + return err + } + + if t, err := handler.TransactionVerify(txnHash); err != nil { + logging.Logger.Error("Failed to verify blobber add/update transaction", zap.Any("err", err), zap.String("txn.Hash", txnHash)) + } else { + logging.Logger.Info("Verified blobber add/update transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) + } + + return err +} + +func setupServerChain() error { + fmt.Print("[6/10] setup server chain") + common.SetupRootContext(node.GetNodeContext()) + + config.SetServerChainID(config.Configuration.ChainID) + serverChain := chain.NewChainFromConfig() + chain.SetServerChain(serverChain) + + if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil { + return err + } + if err := zcncore.SetWalletInfo(node.Self.GetWalletString(), false); err != nil { + return err + } + fmt.Print(" [OK]\n") + return nil +} + +func healthCheckOnChain() error { + txnHash, err := handler.BlobberHealthCheck(common.GetRootContext()) + if err != nil { + if err == handler.ErrBlobberHasRemoved { + return nil + } else { + return err + } + } + + if t, err := handler.TransactionVerify(txnHash); err != nil { + logging.Logger.Error("Failed to verify blobber health check", zap.Any("err", err), zap.String("txn.Hash", txnHash)) + } else { + logging.Logger.Info("Verified blobber health check", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) + } + + return err +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index fd934c5e5..1569386e8 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -90,7 +90,7 @@ func init() { func setup(t *testing.T) { // setup wallet - w, err := zcncrypto.NewBLS0ChainScheme().GenerateKeys() + w, err := zcncrypto.NewSignatureScheme("bls0chain").GenerateKeys() if err != nil { t.Fatal(err) } @@ -301,9 +301,9 @@ func TestHandlers_Requiring_Signature(t *testing.T) { setupEncryptionScheme() router, handlers := setupHandlers() - sch := zcncrypto.NewBLS0ChainScheme() - sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" - _, err := sch.GenerateKeys() + sch := zcncrypto.NewSignatureScheme("bls0chain") + //sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" + _, err := sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") if err != nil { t.Fatal(err) } @@ -2009,8 +2009,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, filePathHash). WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/"), + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/", fileref.CHUNK_SIZE), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). @@ -2156,8 +2156,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, filePathHash). WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder1/subfolder1"), + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder1/subfolder1", filestore.CHUNK_SIZE), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). @@ -2303,8 +2303,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, filePathHash). WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder2/subfolder1"), + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder2/subfolder1", fileref.CHUNK_SIZE), ) rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") diff --git a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go index fc03787af..d429b2e5c 100644 --- a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go @@ -27,7 +27,7 @@ import ( "github.com/0chain/gosdk/core/zcncrypto" ) -const BlobberTestAddr = "localhost:31501" +const BlobberTestAddr = "127.0.0.1:31501" const RetryAttempts = 8 const RetryTimeout = 3 diff --git a/code/go/0chain.net/core/encryption/keys_test.go b/code/go/0chain.net/core/encryption/keys_test.go index 4eb0d206e..7ad0fd4c3 100644 --- a/code/go/0chain.net/core/encryption/keys_test.go +++ b/code/go/0chain.net/core/encryption/keys_test.go @@ -1,13 +1,11 @@ package encryption import ( - "encoding/hex" + "testing" + "github.com/0chain/gosdk/zboxcore/client" "github.com/herumi/bls-go-binary/bls" "github.com/stretchr/testify/require" - "testing" - - "fmt" ) func TestSignatureVerify(t *testing.T) { @@ -62,19 +60,19 @@ func TestDebugOnly(t *testing.T) { // clientKey := "536d2ecfe5aab6c343e8c2e7ee9daa60c43eecc53f4b1c07a6cb2648d9e66c14f2e3fcd43875be40722992f56570fe3c751caacbc7d859b309c787f654bd5a97" // // => 5c2fdfa03fc013cff0e4b716f0529b914e18fd2bc6cdfed49df13b6e3dc4684d - clientKey := "0416c528570ce46eb83584cd604a9ed62644ef4f71a86587d57e4ab91953ff4699107374870799ad4550c4f3833cca2a4d5de75436d67caf89097f1e7d6d7de6d424cb5a08b9dca8957ea7c81a23d066b93a27500954cd29733149ec1f8a8abd540d08f9f81bb24b83ff27e24f173e639573e10a22ed7b0ca326a1aa9dc03e1eef" + //clientKey := "0416c528570ce46eb83584cd604a9ed62644ef4f71a86587d57e4ab91953ff4699107374870799ad4550c4f3833cca2a4d5de75436d67caf89097f1e7d6d7de6d424cb5a08b9dca8957ea7c81a23d066b93a27500954cd29733149ec1f8a8abd540d08f9f81bb24b83ff27e24f173e639573e10a22ed7b0ca326a1aa9dc03e1eef" // => bd3adcacc78ed4352931b138729986a07d2bf0e0a3bf2c885b37a9a0e649dd87 // Looking for bd3adcacc78ed4352931b138729986a07d2bf0e0a3bf2c885b37a9a0e649dd87 - clientKeyBytes, _ := hex.DecodeString(clientKey) - h := Hash(clientKeyBytes) + //clientKeyBytes, _ := hex.DecodeString(clientKey) + // h := Hash(clientKeyBytes) - fmt.Println("hash ", h) + //fmt.Println("hash ", h) - herumipk := MiraclToHerumiPK(clientKey) - fmt.Println("herumipk ", herumipk) - clientKeyBytes2, _ := hex.DecodeString(herumipk) - h = Hash(clientKeyBytes2) - fmt.Println("hash2 ", h) + //herumipk := MiraclToHerumiPK(clientKey) + //fmt.Println("herumipk ", herumipk) + //clientKeyBytes2, _ := hex.DecodeString(herumipk) + // h = Hash(clientKeyBytes2) + //fmt.Println("hash2 ", h) } diff --git a/docker.local/b0docker-compose-github.yml b/docker.local/b0docker-compose-github.yml new file mode 100644 index 000000000..710414d59 --- /dev/null +++ b/docker.local/b0docker-compose-github.yml @@ -0,0 +1,64 @@ +version: "3" +services: + postgres: + image: postgres:11 + environment: + POSTGRES_PORT: 5432 + POSTGRES_HOST: postgres + POSTGRES_USER: postgres + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + volumes: + - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data + networks: + default: + postgres-post: + image: postgres:11 + environment: + POSTGRES_PORT: 5432 + POSTGRES_HOST: postgres + POSTGRES_USER: postgres + volumes: + - ../bin:/blobber/bin + - ../sql:/blobber/sql + command: bash /blobber/bin/postgres-entrypoint.sh + links: + - postgres:postgres + + blobber: + image: blobber + environment: + - DOCKER= true + volumes: + - ../config:/blobber/config + - ./blobber${BLOBBER}/files:/blobber/files + - ./blobber${BLOBBER}/data:/blobber/data + - ./blobber${BLOBBER}/log:/blobber/log + - ./keys_config:/blobber/keysconfig + - ./blobber${BLOBBER}/data/tmp:/tmp + ports: + - 505${BLOBBER}:505${BLOBBER} + - 3150${BLOBBER}:3150${BLOBBER} + command: ./bin/blobber --port 505${BLOBBER} --grpc_port 3150${BLOBBER} --hostname 198.18.0.9${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data --minio_file keysconfig/minio_config.txt + networks: + default: + testnet0: + ipv4_address: 198.18.0.9${BLOBBER} + +networks: + default: + driver: bridge + testnet0: + external: true + # driver: bridge + # ipam: + # config: + # - subnet: 198.18.0.0/15 + # - gateway: 198.18.0.255 + + +volumes: + data: + config: + bin: diff --git a/docker.local/bin/blobber.start_github.sh b/docker.local/bin/blobber.start_github.sh new file mode 100755 index 000000000..8b925a4dd --- /dev/null +++ b/docker.local/bin/blobber.start_github.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +PWD=`pwd` +BLOBBER_DIR=`basename $PWD` +BLOBBER_ID=`echo my directory $BLOBBER_DIR | sed -e 's/.*\(.\)$/\1/'` + + +BLOBBER=$BLOBBER_ID docker-compose -p blobber$BLOBBER_ID -f ../b0docker-compose-github.yml up -d diff --git a/go.mod b/go.mod index e4e679d82..0aee7565e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.3.0-beta.4.0.20211109103416-4d382fe5d5d4 + github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index 081105fd0..922bdb63b 100644 --- a/go.sum +++ b/go.sum @@ -48,35 +48,1499 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.3.0-beta.4.0.20211109103416-4d382fe5d5d4 h1:p5EYDEE4GlAsS01nVwi+pqIU6afoiYqgQHTvRm+US1Y= -github.com/0chain/gosdk v1.3.0-beta.4.0.20211109103416-4d382fe5d5d4/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/0chain/gosdk v1.3.1-0.20211110063152-593138b31ede h1:qd+2f81NaO6htoW8CDIjkgTuZ+BdR7L6MNm/XZ8x4Dw= +github.com/0chain/gosdk v1.3.1-0.20211110063152-593138b31ede/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/0chain/gosdk v1.3.1-0.20211110073225-38b73f0cb34b h1:BekYOJb2ikpNh7dHHTAnNZVWOk0ZRTPph/xiFcnfyKI= +github.com/0chain/gosdk v1.3.1-0.20211110073225-38b73f0cb34b/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/0chain/gosdk v1.3.1-0.20211110104546-f41d127b3ec2 h1:7JvRHUKG+4bJzE4uskq7uKieOQLRQBnAFMW+YQhg/zM= +github.com/0chain/gosdk v1.3.1-0.20211110104546-f41d127b3ec2/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1 h1:7woeGwzIoqGVfVIcEQ2R0S0Ebi9dyBIRCd0zHIdxEuM= +github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= +github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= +github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg= +github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= +github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M= +github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY= +github.com/didip/tollbooth/v6 v6.1.1 h1:Nt7PvWLa9Y94OrykXsFNBinVRQIu8xdy4avpl99Dc1M= github.com/didip/tollbooth/v6 v6.1.1/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.10.11 h1:KKIcwpmur9iTaVbR2dxlHu+peHVhU+/KX//NWvT1n9U= +github.com/ethereum/go-ethereum v1.10.11/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= +github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.55.0 h1:0wVcG9udk2C3TGgmdIGKK9ScOZHZB5nbG+gwji9fhhc= github.com/go-ini/ini v1.55.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc= +github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 h1:rgxjzoDmDXw5q8HONgyHhBas4to0/XWRo/gPpJhsUNQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0/go.mod h1:qrJPVzv9YlhsrxJc3P/Q85nr0w1lIRikTl4JlhdDH5w= +github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= +github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 h1:TV+x87PxW2BcEzpX6yWK0mBEGc9NFppsihFszD6QmyA= github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.6.1/go.mod h1:g8mKMqmSUO6AzAvha7vy07g1rbGOlc7iF0nU0ei83hc= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.4.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.7.1/go.mod h1:nu42q3aPjuC1M0Nak4bnoprKlXPINqopEKqbq5AZSC4= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.9.14 h1:vkPCIhFMn2VdktLUcugqsU4vcLXN3dAhVd1uWA+TDD8= +github.com/klauspost/reedsolomon v1.9.14/go.mod h1:eqPAcE7xar5CIzcdfwydOEdcmchAKAP/qs14y4GCBOk= +github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 h1:R7RAW1p8wjhlHKFhS4X7h8EePqADev/PltCmW9qlJoM= github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20/go.mod h1:sh5SGGmQVGUkWDnxevz0I2FJ4TeC18hRPRjKVBMb2kA= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= +github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magma/augmented-networks/accounting/protos v0.1.1/go.mod h1:Hpfg8aAxldUN7qlVtR5xwlAf8pcetFm8DWwRKZsh2J4= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= +github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= +github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= +github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/selvatico/go-mocket v1.0.7 h1:jbVa7RkoOCzBanQYiYF+VWgySHZogg25fOIKkM38q5k= github.com/selvatico/go-mocket v1.0.7/go.mod h1:7bSWzuNieCdUlanCVu3w0ppS0LvDtPAZmKBIlhoTcp8= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.10+incompatible h1:AL2kpVykjkqeN+MFe1WcwSBVUjGjvdU8/ubvCuXAjrU= +github.com/shirou/gopsutil v3.21.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= +go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= +go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= +go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg= +go.dedis.ch/kyber/v3 v3.0.13 h1:s5Lm8p2/CsTMueQHCN24gPpZ4couBBeKU7r2Yl6r32o= +go.dedis.ch/kyber/v3 v3.0.13/go.mod h1:kXy7p3STAurkADD+/aZcsznZGKVHEqbtmdIzvPfrs1U= +go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRLo= +go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= +go.dedis.ch/protobuf v1.0.11/go.mod h1:97QR256dnkimeNdfmURz0wAMNVbd1VmLXhG1CrTYrJ4= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU= +golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211101204403-39c9dd37992c h1:rnNohYBMnXA07uGnZ9CSWNhIu4Gob4FqWS43lLqZ2sU= +golang.org/x/sys v0.0.0-20211101204403-39c9dd37992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20211101144312-62acf1d99145 h1:vum3nDKdleYb+aePXKFEDT2+ghuH00EgYp9B7Q7EZZE= google.golang.org/genproto v0.0.0-20211101144312-62acf1d99145/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d h1:cEzkQplur9Z++gqjh48MF692Hkdl/jTkbo/7YQ5yssM= gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d/go.mod h1:n2DTgk9at7cr/CWOTKHWPaflj1fN+yuWpdK4lqSUbWA= +gorm.io/driver/mysql v0.3.1 h1:yvUT7Q0I3B9EHJ67NSp6cHbVwcdDHhVUsDAUiFFxRk0= +gorm.io/driver/mysql v0.3.1/go.mod h1:A7H1JD9dKdcjeUTpTuWKEC+E1a74qzW7/zaXqKaTbfM= +gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs= +gorm.io/driver/postgres v1.2.1 h1:JDQKnF7MC51dgL09Vbydc5kl83KkVDlcXfSPJ+xhh68= gorm.io/driver/postgres v1.2.1/go.mod h1:SHRZhu+D0tLOHV5qbxZRUM6kBcf3jp/kxPz2mYMTsNY= +gorm.io/driver/sqlite v1.0.8 h1:omllgSb7/eh9D6lGvLZOdU1ZElxdXuO3dn3Rk+dQxUE= +gorm.io/driver/sqlite v1.0.8/go.mod h1:xkm8/CEmA3yc4zRd0pdCqm43BjO8Hm6avfTpxWb/7c4= +gorm.io/driver/sqlserver v0.2.5 h1:o/MXpn9/BB68RXEEQzfhsSL382yEqUtdCiGIuCspmkY= +gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+odcueLA= +gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.22.0/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.2 h1:1iKcvyJnR5bHydBhDqTwasOkoo6+o4Ms5cknSt6qP7I= gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 735f783b274c05958885d4a9056c4c4195aa3e73 Mon Sep 17 00:00:00 2001 From: stewartie4 Date: Wed, 10 Nov 2021 19:18:38 +0000 Subject: [PATCH 043/161] add system tests workflow --- .github/workflows/system_tests.yml | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/system_tests.yml diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml new file mode 100644 index 000000000..022c5c8f4 --- /dev/null +++ b/.github/workflows/system_tests.yml @@ -0,0 +1,50 @@ +name: "0Chain System Tests" + +on: + workflow_dispatch: + inputs: + zbox_cli_branch: + description: '0Box CLI (branch or commit SHA) which the tests will use' + default: 'staging' + required: true + zwallet_cli_branch: + description: '0Wallet CLI (branch or commit SHA) which the tests will use' + default: 'staging' + required: true + miner_image: + description: 'miner DOCKER IMAGE to deploy' + default: 'staging' + required: true + sharder_image: + description: 'sharder DOCKER IMAGE to deploy' + default: 'staging' + required: true + blobber_image: + description: 'blobber DOCKER IMAGE to deploy' + default: 'staging' + required: true + validator_image: + description: 'validator DOCKER IMAGE to deploy' + default: 'staging' + required: true + zbox_image: + description: '0box DOCKER IMAGE to deploy' + default: 'staging' + required: true + zblock_image: + description: '0block DOCKER IMAGE to deploy' + default: 'staging' + required: true +jobs: + system-tests: + runs-on: [ tests-suite ] + steps: + + - name: Extract branch name + shell: bash + run: | + echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") + echo "##[set-output name=pr_number;]$(echo $pull_number)" + id: extract_branch + From 76098ab417075f99d1c138423b458219ba453195 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 11 Nov 2021 20:32:38 +0800 Subject: [PATCH 044/161] refactor(main):improved startup code (#424) * refactor(main):improved startup code * fix(gomod): fixed go.mod * refactor grpc code Signed-off-by: Kush Trivedi Co-authored-by: Kush Trivedi Co-authored-by: Kush Trivedi <44091822+kushthedude@users.noreply.github.com> --- code/go/0chain.net/blobber/grpc.go | 18 +++++++++++++----- code/go/0chain.net/blobber/http.go | 3 --- code/go/0chain.net/blobber/main.go | 4 ++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go index 04d0462a2..317680544 100644 --- a/code/go/0chain.net/blobber/grpc.go +++ b/code/go/0chain.net/blobber/grpc.go @@ -4,6 +4,7 @@ package main import ( "fmt" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "log" "net" "strconv" @@ -14,17 +15,24 @@ import ( "google.golang.org/grpc/reflection" ) -func startGRPCServer(r mux.Router) { - grpcServer := handler.NewGRPCServerWithMiddlewares(&r) + +func startGRPCServer() { + fmt.Println("[10/10] starting grpc server [OK]") + + r := mux.NewRouter() + + common.ConfigRateLimits() + initHandlers(r) + grpcServer := handler.NewGRPCServerWithMiddlewares(r) reflection.Register(grpcServer) if grpcPort <= 0 { - logging.Logger.Error("Could not start grpc server since grpc port has not been specified." + - " Please specify the grpc port in the --grpc_port build arguement to start the grpc server") + logging.Logger.Error("grpc port missing") return } - logging.Logger.Info("listening too grpc requests on port - " + strconv.Itoa(grpcPort)) + logging.Logger.Info("started grpc server on to grpc requests on port - " + strconv.Itoa(grpcPort)) + lis, err := net.Listen("tcp", fmt.Sprintf(":%v", grpcPort)) if err != nil { log.Fatalf("failed to listen: %v", err) diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index aed7d72bc..03dd564a1 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -66,9 +66,6 @@ func startHttpServer() { logging.Logger.Info("Ready to listen to the requests") - if config.Development() { - go startGRPCServer(*r) - } startTime = time.Now().UTC() diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 06fb1f8d6..1595cb68f 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -44,4 +44,8 @@ func main() { go setupOnChain() startHttpServer() + + startGRPCServer() + + } From 73c141c5923faa4167f9ab5052ec4dcd7ac13c6b Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 11 Nov 2021 20:49:08 +0800 Subject: [PATCH 045/161] start grpc server in background (#428) * start grpc server in background * fix(main): fixed progress bar --- code/go/0chain.net/blobber/config.go | 2 +- code/go/0chain.net/blobber/datastore.go | 2 +- code/go/0chain.net/blobber/filestore.go | 2 +- code/go/0chain.net/blobber/flags.go | 2 +- code/go/0chain.net/blobber/grpc.go | 6 +++--- code/go/0chain.net/blobber/http.go | 3 +-- code/go/0chain.net/blobber/logging.go | 2 +- code/go/0chain.net/blobber/main.go | 4 +++- code/go/0chain.net/blobber/minio.go | 2 +- code/go/0chain.net/blobber/node.go | 2 +- code/go/0chain.net/blobber/zcn.go | 2 +- 11 files changed, 15 insertions(+), 14 deletions(-) diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go index 39d99aac2..2271040f5 100644 --- a/code/go/0chain.net/blobber/config.go +++ b/code/go/0chain.net/blobber/config.go @@ -11,7 +11,7 @@ import ( ) func setupConfig() { - fmt.Print("[2/10] load config") + fmt.Print("[2/11] load config") // setup default config.SetupDefaultConfig() diff --git a/code/go/0chain.net/blobber/datastore.go b/code/go/0chain.net/blobber/datastore.go index bc140634f..dfda9f992 100644 --- a/code/go/0chain.net/blobber/datastore.go +++ b/code/go/0chain.net/blobber/datastore.go @@ -9,7 +9,7 @@ import ( ) func setupDatabase() error { - fmt.Print("\r[7/10] connect data store") + fmt.Print("\r[7/11] connect data store") // check for database connection for i := 0; i < 600; i++ { diff --git a/code/go/0chain.net/blobber/filestore.go b/code/go/0chain.net/blobber/filestore.go index 1c7577e09..7c111af93 100644 --- a/code/go/0chain.net/blobber/filestore.go +++ b/code/go/0chain.net/blobber/filestore.go @@ -9,7 +9,7 @@ import ( var fsStore filestore.FileStore //nolint:unused // global which might be needed somewhere func setupFileStore() (err error) { - fmt.Print("[8/10] setup file store") + fmt.Print("[8/11] setup file store") fsStore, err = filestore.SetupFSStore(filesDir + "/files") diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go index d00a79158..598e1b5e8 100644 --- a/code/go/0chain.net/blobber/flags.go +++ b/code/go/0chain.net/blobber/flags.go @@ -34,7 +34,7 @@ func init() { } func parseFlags() { - fmt.Print("[1/10] load flags") + fmt.Print("[1/11] load flags") flag.Parse() if filesDir == "" { diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go index 317680544..962451b94 100644 --- a/code/go/0chain.net/blobber/grpc.go +++ b/code/go/0chain.net/blobber/grpc.go @@ -4,20 +4,20 @@ package main import ( "fmt" - "github.com/0chain/blobber/code/go/0chain.net/core/common" "log" "net" "strconv" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/gorilla/mux" "google.golang.org/grpc/reflection" ) - func startGRPCServer() { - fmt.Println("[10/10] starting grpc server [OK]") + fmt.Println("[10/11] starting grpc server [OK]") r := mux.NewRouter() diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index 03dd564a1..6bbb932bb 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -23,7 +23,7 @@ import ( var startTime time.Time func startHttpServer() { - fmt.Println("[10/10] start http server [OK]") + fmt.Println("[11/11] start http server [OK]") mode := "main net" if config.Development() { @@ -66,7 +66,6 @@ func startHttpServer() { logging.Logger.Info("Ready to listen to the requests") - startTime = time.Now().UTC() log.Fatal(server.ListenAndServe()) diff --git a/code/go/0chain.net/blobber/logging.go b/code/go/0chain.net/blobber/logging.go index dde22742c..33c449213 100644 --- a/code/go/0chain.net/blobber/logging.go +++ b/code/go/0chain.net/blobber/logging.go @@ -9,7 +9,7 @@ import ( ) func setupLogging() { - fmt.Print("[3/10] init logging") + fmt.Print("[3/11] init logging") if config.Development() { logging.InitLogging("development", logDir, "0chainBlobber.log") diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 1595cb68f..31c7fd36d 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -43,9 +43,11 @@ func main() { go setupOnChain() + go startGRPCServer() + startHttpServer() - startGRPCServer() + } diff --git a/code/go/0chain.net/blobber/minio.go b/code/go/0chain.net/blobber/minio.go index 7b4417448..61d65b482 100644 --- a/code/go/0chain.net/blobber/minio.go +++ b/code/go/0chain.net/blobber/minio.go @@ -10,7 +10,7 @@ import ( ) func setupMinio() error { - fmt.Print("[4/10] setup minio") + fmt.Print("[4/11] setup minio") reader, err := os.Open(minioFile) if err != nil { return err diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index 710a68ed9..2d37fa9d8 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -12,7 +12,7 @@ import ( ) func setupNode() error { - fmt.Print("[5/10] setup blobber") + fmt.Print("[5/11] setup blobber") reader, err := os.Open(keysFile) if err != nil { diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index 4023dff1f..d602d727e 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -15,7 +15,7 @@ import ( ) func setupOnChain() { - fmt.Println("[9/10] connecting to chain ") + fmt.Println("[9/11] connecting to chain ") const ATTEMPT_DELAY = 60 * 1 From 5b0a7dd86d14b25a4b8f82f62b592c8705837d88 Mon Sep 17 00:00:00 2001 From: Andika Demas Riyandi Date: Fri, 12 Nov 2021 09:39:34 +0700 Subject: [PATCH 046/161] =?UTF-8?q?move=20herumi=20installation=20to=20bas?= =?UTF-8?q?e=20image=20and=20use=20it=20to=20the=20entire=20docke=E2=80=A6?= =?UTF-8?q?=20(#422)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * move herumi installation to base image and use it to the entire dockerfile * update go.sum * update build publish docker image in github actions Co-authored-by: Lz --- .../build-&-publish-docker-image.yml | 39 +++++++++---------- docker.aws/build.blobber/Dockerfile | 2 +- docker.aws/build.validator/Dockerfile | 2 +- docker.local/Dockerfile | 19 +-------- docker.local/Dockerfile.dev | 19 +-------- .../IntegrationTestsBlobberDockerfile | 15 +------ docker.local/ValidatorDockerfile | 19 +-------- docker.local/ValidatorDockerfile.dev | 19 +-------- docker.local/base.Dockerfile | 12 ++---- docker.local/bin/clean.sh | 3 +- go.sum | 6 --- 11 files changed, 31 insertions(+), 124 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index d0ccae868..855f531ba 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -30,24 +30,22 @@ jobs: echo ::set-output name=BRANCH::${BRANCH} echo ::set-output name=VERSION::${BRANCH}-${SHORT_SHA} + - name: Build base + run: ./docker.local/bin/build.base.sh + + - name: Build blobber + run: ./docker.local/bin/build.blobber.sh + - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build blobber Docker Image - run: | - docker build -t $BLOBBER_REGISTRY:$TAG -f "$DOCKERFILE_BLOB" . - docker tag $BLOBBER_REGISTRY:$TAG $BLOBBER_REGISTRY:latest - docker tag $BLOBBER_REGISTRY:$TAG $BLOBBER_REGISTRY:staging - docker push $BLOBBER_REGISTRY:$TAG - env: - TAG: ${{ steps.get_version.outputs.VERSION }} - DOCKERFILE_BLOB: "docker.local/Dockerfile" - - name: Push blobber Docker Image run: | + docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG + docker push ${BLOBBER_REGISTRY}:$TAG if [[ "$PUSH_LATEST" == "yes" ]] && [[ "$BRANCH" == "master" ]]; then docker push $BLOBBER_REGISTRY:latest elif [[ "$BRANCH" == "staging" ]]; then @@ -55,6 +53,7 @@ jobs: fi env: PUSH_LATEST: ${{ github.event.inputs.latest_tag }} + TAG: ${{ steps.get_version.outputs.VERSION }} BRANCH: ${{ steps.get_version.outputs.BRANCH }} dockerize_validator: @@ -69,24 +68,23 @@ jobs: SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) echo ::set-output name=BRANCH::${BRANCH} echo ::set-output name=VERSION::${BRANCH}-${SHORT_SHA} + + - name: Build base + run: ./docker.local/bin/build.base.sh + + - name: Build validator + run: ./docker.local/bin/build.validator.sh + - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build validator Docker Image - run: | - docker build -t $VALIDATOR_REGISTRY:$TAG -f "$DOCKERFILE_PROXY" . - docker tag $VALIDATOR_REGISTRY:$TAG $VALIDATOR_REGISTRY:latest - docker tag $VALIDATOR_REGISTRY:$TAG $VALIDATOR_REGISTRY:staging - docker push $VALIDATOR_REGISTRY:$TAG - env: - TAG: ${{ steps.get_version.outputs.VERSION }} - DOCKERFILE_PROXY: "docker.local/ValidatorDockerfile" - - name: Push validator Docker Image run: | + docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG + docker push ${VALIDATOR_REGISTRY}:$TAG if [[ "$PUSH_LATEST" == "yes" ]] && [[ "$BRANCH" == "master" ]]; then docker push $VALIDATOR_REGISTRY:latest elif [[ "$BRANCH" == "staging" ]]; then @@ -94,4 +92,5 @@ jobs: fi env: PUSH_LATEST: ${{ github.event.inputs.latest_tag }} + TAG: ${{ steps.get_version.outputs.VERSION }} BRANCH: ${{ steps.get_version.outputs.BRANCH }} diff --git a/docker.aws/build.blobber/Dockerfile b/docker.aws/build.blobber/Dockerfile index 38ba7d7e4..e87a08b0f 100644 --- a/docker.aws/build.blobber/Dockerfile +++ b/docker.aws/build.blobber/Dockerfile @@ -12,7 +12,7 @@ RUN apk add gmp gmp-dev openssl-dev && \ mv bls* bls && \ make -C mcl -j $(nproc) lib/libmclbn256.so install && \ cp mcl/lib/libmclbn256.so /usr/local/lib && \ - make -C bls -j $(nproc) install && \ + make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \ rm -R /tmp/mcl && \ rm -R /tmp/bls diff --git a/docker.aws/build.validator/Dockerfile b/docker.aws/build.validator/Dockerfile index 122aee4aa..bb8652a29 100644 --- a/docker.aws/build.validator/Dockerfile +++ b/docker.aws/build.validator/Dockerfile @@ -12,7 +12,7 @@ RUN apk add gmp gmp-dev openssl-dev && \ mv bls* bls && \ make -C mcl -j $(nproc) lib/libmclbn256.so install && \ cp mcl/lib/libmclbn256.so /usr/local/lib && \ - make -C bls -j $(nproc) install && \ + make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \ rm -R /tmp/mcl && \ rm -R /tmp/bls diff --git a/docker.local/Dockerfile b/docker.local/Dockerfile index 17da9f577..4fdc70a96 100644 --- a/docker.local/Dockerfile +++ b/docker.local/Dockerfile @@ -1,26 +1,9 @@ -FROM golang:1.17.1-alpine3.14 as blobber_build +FROM blobber_base as blobber_build LABEL zchain="blobber" RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep -# Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev && \ - cd /tmp && \ - wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && \ - make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib && \ - rm -R /tmp/mcl -#TODO: create shared image and remove code duplicates! -RUN git clone https://github.com/herumi/bls /tmp/bls && \ - cd /tmp/bls && \ - git submodule init && \ - git submodule update && \ - make -j $(nproc) install && \ - cd - && \ - rm -R /tmp/bls - ENV SRC_DIR=/blobber ENV GO111MODULE=on diff --git a/docker.local/Dockerfile.dev b/docker.local/Dockerfile.dev index d65dd778b..a03130e6d 100644 --- a/docker.local/Dockerfile.dev +++ b/docker.local/Dockerfile.dev @@ -1,24 +1,7 @@ -FROM golang:1.17.1-alpine3.14 as blobber_build +FROM blobber_base as blobber_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep -# Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev && \ - cd /tmp && \ - wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && \ - make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib && \ - rm -R /tmp/mcl -#TODO: create shared image and remove code duplicates! -RUN git clone https://github.com/herumi/bls /tmp/bls && \ - cd /tmp/bls && \ - git submodule init && \ - git submodule update && \ - make -j $(nproc) install && \ - cd - && \ - rm -R /tmp/bls - ENV SRC_DIR=/blobber ENV GO111MODULE=on diff --git a/docker.local/IntegrationTestsBlobberDockerfile b/docker.local/IntegrationTestsBlobberDockerfile index a6a132008..4d82f8c57 100644 --- a/docker.local/IntegrationTestsBlobberDockerfile +++ b/docker.local/IntegrationTestsBlobberDockerfile @@ -1,20 +1,7 @@ -FROM golang:1.17.1-alpine3.14 as blobber_build +FROM blobber_base as blobber_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep -# Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev && \ - cd /tmp && \ - wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ - wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && \ - mv bls* bls && \ - make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib && \ - make -C bls -j $(nproc) install && \ - rm -R /tmp/mcl && \ - rm -R /tmp/bls - ENV SRC_DIR=/blobber ENV GO111MODULE=on diff --git a/docker.local/ValidatorDockerfile b/docker.local/ValidatorDockerfile index db2a19b6c..0dd54dd25 100644 --- a/docker.local/ValidatorDockerfile +++ b/docker.local/ValidatorDockerfile @@ -1,26 +1,9 @@ -FROM golang:1.17.1-alpine3.14 as validator_build +FROM blobber_base as validator_build LABEL zchain="validator" RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep -# Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev && \ - cd /tmp && \ - wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && \ - make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib && \ - rm -R /tmp/mcl - -RUN git clone https://github.com/herumi/bls /tmp/bls && \ - cd /tmp/bls && \ - git submodule init && \ - git submodule update && \ - make -j $(nproc) install && \ - cd - && \ - rm -R /tmp/bls - ENV SRC_DIR=/blobber ENV GO111MODULE=on diff --git a/docker.local/ValidatorDockerfile.dev b/docker.local/ValidatorDockerfile.dev index ab2a10953..34ee55f50 100644 --- a/docker.local/ValidatorDockerfile.dev +++ b/docker.local/ValidatorDockerfile.dev @@ -1,24 +1,7 @@ -FROM golang:1.17.1-alpine3.14 as validator_build +FROM blobber_base as validator_build RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep -# Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev && \ - cd /tmp && \ - wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && \ - make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib && \ - rm -R /tmp/mcl - -RUN git clone https://github.com/herumi/bls /tmp/bls && \ - cd /tmp/bls && \ - git submodule init && \ - git submodule update && \ - make -j $(nproc) install && \ - cd - && \ - rm -R /tmp/bls - ENV SRC_DIR=/blobber ENV GO111MODULE=on diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 0d9429d9d..36a620531 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -10,15 +10,11 @@ RUN apk add --update --no-cache build-base linux-headers git cmake bash perl gre RUN apk add gmp gmp-dev openssl-dev && \ cd /tmp && \ wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ + wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \ mv mcl* mcl && \ + mv bls* bls && \ make -C mcl -j $(nproc) lib/libmclbn256.so install && \ cp mcl/lib/libmclbn256.so /usr/local/lib && \ - rm -R /tmp/mcl - -RUN git clone https://github.com/herumi/bls /tmp/bls && \ - cd /tmp/bls && \ - git submodule init && \ - git submodule update && \ - make -j $(nproc) install && \ - cd - && \ + make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \ + rm -R /tmp/mcl && \ rm -R /tmp/bls diff --git a/docker.local/bin/clean.sh b/docker.local/bin/clean.sh index c216a40aa..8f3280bb8 100755 --- a/docker.local/bin/clean.sh +++ b/docker.local/bin/clean.sh @@ -2,8 +2,7 @@ for i in $(seq 1 6) do - rm docker.local/blobber$i/log/* + rm -rf docker.local/blobber$i/log/* rm -rf docker.local/blobber$i/data/postgresql/* rm -rf docker.local/blobber$i/files/* done - diff --git a/go.sum b/go.sum index 922bdb63b..9c1dbe1cc 100644 --- a/go.sum +++ b/go.sum @@ -48,12 +48,6 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.3.1-0.20211110063152-593138b31ede h1:qd+2f81NaO6htoW8CDIjkgTuZ+BdR7L6MNm/XZ8x4Dw= -github.com/0chain/gosdk v1.3.1-0.20211110063152-593138b31ede/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= -github.com/0chain/gosdk v1.3.1-0.20211110073225-38b73f0cb34b h1:BekYOJb2ikpNh7dHHTAnNZVWOk0ZRTPph/xiFcnfyKI= -github.com/0chain/gosdk v1.3.1-0.20211110073225-38b73f0cb34b/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= -github.com/0chain/gosdk v1.3.1-0.20211110104546-f41d127b3ec2 h1:7JvRHUKG+4bJzE4uskq7uKieOQLRQBnAFMW+YQhg/zM= -github.com/0chain/gosdk v1.3.1-0.20211110104546-f41d127b3ec2/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1 h1:7woeGwzIoqGVfVIcEQ2R0S0Ebi9dyBIRCd0zHIdxEuM= github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= From 765ecb548d3bfcccbe105acdf43ff8cf6b14ffae Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 12 Nov 2021 10:47:05 +0800 Subject: [PATCH 047/161] fix(devops): fixed git clone with checkout@v1 --- .github/workflows/build-&-publish-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 855f531ba..63079271d 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -20,7 +20,7 @@ jobs: dockerize_blobber: runs-on: [self-hosted, build] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v1 - name: Get the version id: get_version From 8a4661d6bc8e0e4705ad44f7846086cb669b42c1 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 12 Nov 2021 11:06:01 +0800 Subject: [PATCH 048/161] fix(devops):merged dockerize.yml into build-&-publish-docker-image.yml --- .../build-&-publish-docker-image.yml | 259 ++++++++++++------ .github/workflows/dockerize.yml | 208 -------------- 2 files changed, 176 insertions(+), 291 deletions(-) delete mode 100644 .github/workflows/dockerize.yml diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 63079271d..f94748db1 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -1,96 +1,189 @@ -name: build-&-publish-docker-image +name: Dockerize on: - push: - branches: [ staging ] + push: + branches: + - staging + tags: + - '*' + workflow_dispatch: - inputs: - latest_tag: - description: 'type yes for building latest tag' - default: 'no' - required: true + env: - ZCHAIN_BUILDBASE: zchain_build_base - ZCHAIN_BUILDRUN: zchain_run_base BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} jobs: - dockerize_blobber: + build_base: runs-on: [self-hosted, build] steps: - - uses: actions/checkout@v1 - - - name: Get the version - id: get_version - run: | - BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g') - SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) - echo ::set-output name=BRANCH::${BRANCH} - echo ::set-output name=VERSION::${BRANCH}-${SHORT_SHA} - - - name: Build base - run: ./docker.local/bin/build.base.sh - - - name: Build blobber - run: ./docker.local/bin/build.blobber.sh - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Push blobber Docker Image - run: | - docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG - docker push ${BLOBBER_REGISTRY}:$TAG - if [[ "$PUSH_LATEST" == "yes" ]] && [[ "$BRANCH" == "master" ]]; then - docker push $BLOBBER_REGISTRY:latest - elif [[ "$BRANCH" == "staging" ]]; then - docker push $BLOBBER_REGISTRY:staging - fi - env: - PUSH_LATEST: ${{ github.event.inputs.latest_tag }} - TAG: ${{ steps.get_version.outputs.VERSION }} - BRANCH: ${{ steps.get_version.outputs.BRANCH }} - - dockerize_validator: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + - name: Clone blobber + uses: actions/checkout@v1 + + - name: Build blobber_base + run: ./docker.local/bin/build.base.sh + + - name: Export blobber_base + run: | + mkdir -p /tmp/0chain/ + docker save "blobber_base" > /tmp/0chain/blobber_base.tar + + - name: Upload blobber_base + uses: actions/upload-artifact@v2 + with: + name: blobber_base + path: /tmp/0chain/blobber_base.tar + + build_blobber: + needs: build_base runs-on: [self-hosted, build] steps: - - uses: actions/checkout@v1 - - - name: Get the version - id: get_version - run: | - BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g') - SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) - echo ::set-output name=BRANCH::${BRANCH} - echo ::set-output name=VERSION::${BRANCH}-${SHORT_SHA} - - - name: Build base - run: ./docker.local/bin/build.base.sh - - - name: Build validator - run: ./docker.local/bin/build.validator.sh - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Push validator Docker Image - run: | - docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG - docker push ${VALIDATOR_REGISTRY}:$TAG - if [[ "$PUSH_LATEST" == "yes" ]] && [[ "$BRANCH" == "master" ]]; then - docker push $VALIDATOR_REGISTRY:latest - elif [[ "$BRANCH" == "staging" ]]; then - docker push $VALIDATOR_REGISTRY:staging - fi - env: - PUSH_LATEST: ${{ github.event.inputs.latest_tag }} - TAG: ${{ steps.get_version.outputs.VERSION }} - BRANCH: ${{ steps.get_version.outputs.BRANCH }} + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Clone blobber + uses: actions/checkout@v1 + + + - name: Download blobber_base + uses: actions/download-artifact@v2 + with: + name: blobber_base + path: /tmp/0chain + + - name: Load blobber_base + run: | + docker load --input /tmp/0chain/blobber_base.tar + + - name: Build blobber + run: ./docker.local/bin/build.blobber.sh + + - name: Export blobber + run: | + mkdir -p /tmp/0chain/ + docker save "blobber" > /tmp/0chain/blobber.tar + + - name: Upload blobber + uses: actions/upload-artifact@v2 + with: + name: blobber + path: /tmp/0chain/blobber.tar + + publish_blobber: + needs: build_blobber + runs-on: [self-hosted] + steps: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Download blobber + uses: actions/download-artifact@v2 + with: + name: blobber + path: /tmp/0chain + + - name: Load blobber + run: | + docker load --input /tmp/0chain/blobber.tar + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push blobber + run: | + docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG + docker push ${BLOBBER_REGISTRY}:$TAG + + SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) + docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA + docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA + + + + build_validator: + needs: build_base + runs-on: [self-hosted, build] + steps: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Clone blobber + uses: actions/checkout@v1 + + + - name: Download blobber_base + uses: actions/download-artifact@v2 + with: + name: blobber_base + path: /tmp/0chain + + - name: Load blobber_base + run: | + docker load --input /tmp/0chain/blobber_base.tar + + #docker image ls -a + + - name: Build validator + run: ./docker.local/bin/build.validator.sh + + - name: Export validator + run: | + mkdir -p /tmp/0chain/ + docker save "validator" > /tmp/0chain/validator.tar + + - name: Upload validator + uses: actions/upload-artifact@v2 + with: + name: validator + path: /tmp/0chain/validator.tar + + publish_validator: + needs: build_validator + runs-on: [self-hosted] + steps: + - name: Set GITHUB_ENV + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV + echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + + - name: Download validator + uses: actions/download-artifact@v2 + with: + name: validator + path: /tmp/0chain + + - name: Load validator + run: | + docker load --input /tmp/0chain/validator.tar + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push validator + run: | + docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG + docker push ${VALIDATOR_REGISTRY}:$TAG + + SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) + docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA + docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml deleted file mode 100644 index aee7d33c3..000000000 --- a/.github/workflows/dockerize.yml +++ /dev/null @@ -1,208 +0,0 @@ -name: Dockerize - -on: - push: - branches: - - staging - - stream - tags: - - '*' - - -env: - BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} - VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} - -jobs: - build_base: - runs-on: [self-hosted, build] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - name: Clone blobber - uses: actions/checkout@v1 - - # - name: Clone gosdk - # run: | - # [ -d $GOSDK ] && rm -rf $GOSDK - # git clone https://github.com/0chain/gosdk.git $GOSDK - # cd $GOSDK - # git checkout $BRANCH - - - - name: Build blobber_base - run: ./docker.local/bin/build.base.sh - - - name: Export blobber_base - run: | - mkdir -p /tmp/0chain/ - docker save "blobber_base" > /tmp/0chain/blobber_base.tar - - - name: Upload blobber_base - uses: actions/upload-artifact@v2 - with: - name: blobber_base - path: /tmp/0chain/blobber_base.tar - - build_blobber: - needs: build_base - runs-on: [self-hosted, build] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - - name: Clone blobber - uses: actions/checkout@v1 - - # - name: Clone gosdk - # run: | - # [ -d $GOSDK ] && rm -rf $GOSDK - # git clone https://github.com/0chain/gosdk.git $GOSDK - # cd $GOSDK - # git checkout $BRANCH - - - name: Download blobber_base - uses: actions/download-artifact@v2 - with: - name: blobber_base - path: /tmp/0chain - - - name: Load blobber_base - run: | - docker load --input /tmp/0chain/blobber_base.tar - - - name: Build blobber - run: ./docker.local/bin/build.blobber.sh - - - name: Export blobber - run: | - mkdir -p /tmp/0chain/ - docker save "blobber" > /tmp/0chain/blobber.tar - - - name: Upload blobber - uses: actions/upload-artifact@v2 - with: - name: blobber - path: /tmp/0chain/blobber.tar - - publish_blobber: - needs: build_blobber - runs-on: [self-hosted] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - - name: Download blobber - uses: actions/download-artifact@v2 - with: - name: blobber - path: /tmp/0chain - - - name: Load blobber - run: | - docker load --input /tmp/0chain/blobber.tar - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push blobber - run: | - docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG - docker push ${BLOBBER_REGISTRY}:$TAG - - SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) - docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA - docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA - - - - build_validator: - needs: build_base - runs-on: [self-hosted, build] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - - name: Clone blobber - uses: actions/checkout@v1 - - # - name: Clone gosdk - # run: | - # [ -d $GOSDK ] && rm -rf $GOSDK - # git clone https://github.com/0chain/gosdk.git $GOSDK - # cd $GOSDK - # git checkout $BRANCH - - - name: Download blobber_base - uses: actions/download-artifact@v2 - with: - name: blobber_base - path: /tmp/0chain - - - name: Load blobber_base - run: | - docker load --input /tmp/0chain/blobber_base.tar - - #docker image ls -a - - - name: Build validator - run: ./docker.local/bin/build.validator.sh - - - name: Export validator - run: | - mkdir -p /tmp/0chain/ - docker save "validator" > /tmp/0chain/validator.tar - - - name: Upload validator - uses: actions/upload-artifact@v2 - with: - name: validator - path: /tmp/0chain/validator.tar - - publish_validator: - needs: build_validator - runs-on: [self-hosted] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - - name: Download validator - uses: actions/download-artifact@v2 - with: - name: validator - path: /tmp/0chain - - - name: Load validator - run: | - docker load --input /tmp/0chain/validator.tar - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push validator - run: | - docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG - docker push ${VALIDATOR_REGISTRY}:$TAG - - SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) - docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA - docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA From 7ff2986c71da3b870226940d813e7a172f066e3b Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 12 Nov 2021 11:10:40 +0800 Subject: [PATCH 049/161] fix(devops):merged ci.yml into tests.yml --- .github/workflows/ci.yml | 129 ------------------------------------ .github/workflows/tests.yml | 81 ++++++++++++++++++---- 2 files changed, 69 insertions(+), 141 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index fc79a8d72..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: CI - -on: - push: - branches: [ master, staging ] - - release: - types: - - published - pull_request: - paths-ignore: - - '**.md' - -env: - BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} - VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} - -jobs: - integration_test: - runs-on: [ubuntu-20.04] - timeout-minutes: 40 - steps: - - - name: Setup go 1.17 - uses: actions/setup-go@v2 - with: - go-version: '1.17' # The Go version to download (if necessary) and use. - - - name: Clone Blobber - uses: actions/checkout@v1 - - - name: Build Blobber - run: | - docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 - ./docker.local/bin/build.base.sh - ./docker.local/bin/build.blobber.sh - - - name: Run Blobber on gRPC - run: | - ./docker.local/bin/blobber.init.setup.sh - cd docker.local/blobber1 - ../bin/blobber.start_github.sh - docker ps - - - name: Run Tests - run: | - go17=$(which go) - sudo $go17 test -tags bn256 ./... -args integration - - golangci: - name: lint - runs-on: [self-hosted, tests-suite] - steps: - - name: Setup go 1.16 - uses: actions/setup-go@v2 - with: - go-version: '1.16' # The Go version to download (if necessary) and use. - - uses: actions/checkout@v2 - - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - with: - version: latest - args: --timeout=2m0s - - buf-build: - runs-on: [self-hosted, tests-suite] - steps: - - name: Check out code - uses: actions/checkout@master - with: - fetch-depth: 1 - - - name: setup golang - if: success() - uses: actions/setup-go@v2 - with: - go-version: 1.16.x - - name: run buf commands - if: success() - run: make local - - dockerize_blobber: - runs-on: [self-hosted, build] - needs: integration_test - if: github.event_name == 'release' && github.event.action == 'published' - steps: - - uses: actions/checkout@v2 - - - name: Get the version - id: get_version - run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3) - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build - run: | - docker build -t $BLOBBER_REGISTRY:$TAG -f docker.local/Dockerfile . - docker push $BLOBBER_REGISTRY:$TAG - env: - TAG: ${{ steps.get_version.outputs.VERSION }} - - dockerize_validator: - runs-on: [self-hosted, build] - needs: integration_test - if: github.event_name == 'release' && github.event.action == 'published' - steps: - - uses: actions/checkout@v2 - - - name: Get the version - id: get_version - run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3) - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build - run: | - docker build -t $VALIDATOR_REGISTRY:$TAG -f docker.local/ValidatorDockerfile . - docker push $VALIDATOR_REGISTRY:$TAG - env: - TAG: ${{ steps.get_version.outputs.VERSION }} \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a91b0f5d0..901edfe92 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,15 +6,17 @@ on: - master - staging pull_request: + paths-ignore: + - '**.md' jobs: unit_tests: runs-on: [self-hosted, load-test] steps: - - name: Setup go 1.16 + - name: Setup go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' # The Go version to download (if necessary) and use. + go-version: '1.17' # The Go version to download (if necessary) and use. - name: Clone blobber uses: actions/checkout@v2 @@ -25,18 +27,73 @@ jobs: # echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV # echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - # - name: Clone gosdk - # run: | - # [ -d $GOSDK ] && rm -rf $GOSDK - # git clone https://github.com/0chain/gosdk.git $GOSDK - # cd $GOSDK - # git checkout $BRANCH - name: Run tests run: | cd $GITHUB_WORKSPACE/code/go/0chain.net/ CGO_ENABLED=1 go test -tags bn256 -race -coverprofile=coverage.txt -covermode=atomic ./... - # - name: Codecov - # run: | - # cd $GITHUB_WORKSPACE/code/go/0chain.net/ - # bash <(curl -s https://codecov.io/bash) \ No newline at end of file + + + integration_test: + runs-on: [ubuntu-20.04] + timeout-minutes: 40 + steps: + + - name: Setup go 1.17 + uses: actions/setup-go@v2 + with: + go-version: '1.17' # The Go version to download (if necessary) and use. + + - name: Clone Blobber + uses: actions/checkout@v1 + + - name: Build Blobber + run: | + docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 + ./docker.local/bin/build.base.sh + ./docker.local/bin/build.blobber.sh + + - name: Run Blobber on gRPC + run: | + ./docker.local/bin/blobber.init.setup.sh + cd docker.local/blobber1 + ../bin/blobber.start_github.sh + docker ps + + - name: Run Tests + run: | + go17=$(which go) + sudo CGO_ENABLED=1 $go17 test -tags bn256 ./... -args integration + + golangci: + name: lint + runs-on: [self-hosted, tests-suite] + steps: + - name: Setup go 1.17 + uses: actions/setup-go@v2 + with: + go-version: '1.17' # The Go version to download (if necessary) and use. + - uses: actions/checkout@v2 + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: --timeout=2m0s + + buf-build: + runs-on: [self-hosted, tests-suite] + steps: + - name: Check out code + uses: actions/checkout@master + with: + fetch-depth: 1 + + - name: setup golang + if: success() + uses: actions/setup-go@v2 + with: + go-version: 1.17.x + - name: run buf commands + if: success() + run: make local \ No newline at end of file From 116d592ecad26cf7012b6d6bd8ba097965f4b787 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 12 Nov 2021 11:25:09 +0800 Subject: [PATCH 050/161] fix(devops):added multiple mirrors for apline package repositories --- docker.local/base.Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 36a620531..583b31c20 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -4,6 +4,22 @@ LABEL zchain="blobber" # RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://mirrors.aliyun.com/alpine/v3.14/community" >> /etc/apk/repositories + +RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/community" >> /etc/apk/repositories + +RUN echo "https://uk.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories + +RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories + +RUN echo "https://mirror.yandex.ru/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://mirror.yandex.ru/alpine/v3.14/community" >> /etc/apk/repositories + + RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep # Install Herumi's cryptography From 912f094788d6588fbf3457d5a62564d3417bc3c2 Mon Sep 17 00:00:00 2001 From: Andika Demas Riyandi Date: Tue, 16 Nov 2021 04:36:11 +0700 Subject: [PATCH 051/161] fix script for linux build (#433) --- dev.local/cli.sh | 55 +++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/dev.local/cli.sh b/dev.local/cli.sh index a6e026d51..f237ca2ea 100755 --- a/dev.local/cli.sh +++ b/dev.local/cli.sh @@ -88,7 +88,6 @@ install_debuggger() { } cleanAll() { - cd $root rm -rf ./data && echo "data is removed" } @@ -112,31 +111,36 @@ done install_postgres () { - echo Installing blobber_postgres in docker... [ ! "$(docker ps -a | grep blobber_postgres)" ] && docker run --name blobber_postgres --restart always -p 5432:5432 -e POSTGRES_PASSWORD=postgres -d postgres:11 - [ -d "./data/blobber" ] || mkdir -p "./data/blobber" - echo Initializing database + [ -d "./data/blobber$i" ] || mkdir -p "./data/blobber$i" - [ -d "./data/blobber/sql" ] && rm -rf [ -d "./data/blobber/sql" ] + echo Initializing database - cp -r ../sql "./data/blobber/" - cd "./data/blobber/sql" + [ -d "./data/blobber$i/sql" ] && rm -rf [ -d "./data/blobber$i/sql" ] - find . -name "*.sql" -exec sed -i '' "s/blobber_user/blobber_user$i/g" {} \; - find . -name "*.sql" -exec sed -i '' "s/blobber_meta/blobber_meta$i/g" {} \; + cp -r ../sql "./data/blobber$i/" + cd "./data/blobber$i/sql" + if [[ "$(uname)" == "Darwin" ]] + then + find . -name "*.sql" -exec sed -i '' "s/blobber_user/blobber_user$i/g" {} \; + find . -name "*.sql" -exec sed -i '' "s/blobber_meta/blobber_meta$i/g" {} \; + else + sed -i "s/blobber_user/blobber_user$i/g" *.sql; + sed -i "s/blobber_meta/blobber_meta$i/g" *.sql; + fi cd $root - [ -d "./data/blobber/bin" ] && rm -rf [ -d "./data/blobber/bin" ] - cp -r ../bin "./data/blobber/" + [ -d "./data/blobber$i/bin" ] && rm -rf [ -d "./data/blobber$i/bin" ] + cp -r ../bin "./data/blobber$i/" cd $root - + [ ! "$(docker ps -a | grep blobber_postgres_init)" ] && docker rm blobber_postgres_init --force @@ -146,25 +150,30 @@ install_postgres () { -e POSTGRES_HOST=postgres \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ - -v $root/data/blobber/bin:/blobber/bin \ - -v $root/data/blobber/sql:/blobber/sql \ + -v $root/data/blobber$i/bin:/blobber/bin \ + -v $root/data/blobber$i/sql:/blobber/sql \ postgres:11 bash /blobber/bin/postgres-entrypoint.sh docker rm blobber_postgres_init --force - } prepareRuntime() { - cd $root [ -d ./data/blobber$i/config ] && rm -rf $root/data/blobber$i/config cp -r ../config "./data/blobber$i/" cd ./data/blobber$i/config/ + if [[ "$(uname)" == "Darwin" ]] + then + find . -name "*.yaml" -exec sed -i '' "s/blobber_user/blobber_user$i/g" {} \; + find . -name "*.yaml" -exec sed -i '' "s/blobber_meta/blobber_meta$i/g" {} \; + find . -name "*.yaml" -exec sed -i '' "s/postgres/127.0.0.1/g" {} \; + else + sed -i "s/blobber_user/blobber_user$i/g" *.yaml; + sed -i "s/blobber_meta/blobber_meta$i/g" *.yaml; + sed -i "s/postgres/127.0.0.1/g" *.yaml + fi - find . -name "*.yaml" -exec sed -i '' "s/blobber_user/blobber_user$i/g" {} \; - find . -name "*.yaml" -exec sed -i '' "s/blobber_meta/blobber_meta$i/g" {} \; - find . -name "*.yaml" -exec sed -i '' "s/postgres/127.0.0.1/g" {} \; cd $root/data/blobber$i/ [ -d files ] || mkdir files @@ -174,7 +183,6 @@ prepareRuntime() { start_blobber () { - echo ">>>>>>>>>>>>>> Blobber $i <<<<<<<<<<<<<<<<" echo "[1/3] build blobber..." @@ -196,12 +204,10 @@ start_blobber () { echo "[3/3] run blobber..." - ./data/blobber$i/blobber --port $port --grpc_port $grpc_port --hostname $hostname --deployment_mode 0 --keys_file $keys_file --files_dir $files_dir --log_dir $log_dir --db_dir $db_dir --minio_file $minio_file --config_dir $config_dir } start_validator () { - echo ">>>>>>>>>>>>>> Validator $i <<<<<<<<<<<<<<<<" echo "[1/3] build validator..." @@ -223,14 +229,11 @@ start_validator () { } clean () { - echo "Building blobber $i" - + echo "Cleaning blobber $i" cd $root - rm -rf "./data/blobber$i" } - echo " ********************************************** Blobber/Validator $i From d947df0a6a8d8f8c2ddf73a31464bb707d8c268d Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 16 Nov 2021 05:39:53 +0800 Subject: [PATCH 052/161] Fix/integration tests GitHub (#429) * fix(devops): added gRPC status checking * fix(devops): enabled tesets on pr and tags * fix(devops): fixed yml syntax issues * fix(devops): fixed yml syntax issues * fix(devops): fixed yml syntax issues * fix(main): fixed grpc server startup bug * fix(dockerfile):uncomment local GOPROXY * fix(devops):run integration-tests in single thread * fix(devops):enabled integration switch * feat(devops): run blobber outside docker on integration-tests job * fix(devops): fixed sed issue on linux * fix(devops): skip to setup on chain if it is integration-test on github * fix(devops): merged Run blobber and Run tests * fix(devops): moved integration variable to env * fix(devops): fixed panic issue if zcn is offline on integration-tests * fix(devops): reverted uncommentted code for debugging * fix(devops): fixed evn vars for Run Tests * fix(devops): fixed use make command instead of script * fix(devops): run make integration-tests as sudoer * fix(devops): use custom go to build it * fix(devops): sleep more time to wait grpc server to up * fix(devops): added status checking * fix(devops): added sleep time for build * fix(devops): removed local-build from local-run * fix(devops): run make local-run with sudoer * fix(devops): disable workers on integration-tests * fix(devops): enabled dockerize job on PR for system_test * fix(devops): fixed GITHUB_REF for PR --- .../build-&-publish-docker-image.yml | 6 +- .github/workflows/tests.yml | 39 ++++++------ Makefile | 45 +++++++++++-- code/go/0chain.net/blobber/datastore.go | 2 +- code/go/0chain.net/blobber/flags.go | 24 ++++--- code/go/0chain.net/blobber/grpc.go | 15 +++-- code/go/0chain.net/blobber/http.go | 6 +- code/go/0chain.net/blobber/main.go | 7 +-- code/go/0chain.net/blobber/zcn.go | 63 ++++++++++++------- .../handler/download_integration_test.go | 16 ++--- .../handler/helper_integration_test.go | 12 ++-- docker.local/b0docker-compose-github.yml | 20 ------ docker.local/blobber.Dockerfile | 2 +- 13 files changed, 150 insertions(+), 107 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index f94748db1..67c154362 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -6,6 +6,7 @@ on: - staging tags: - '*' + pull_request: workflow_dispatch: @@ -20,9 +21,8 @@ jobs: steps: - name: Set GITHUB_ENV run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 901edfe92..24b54c7d7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,12 +5,13 @@ on: branches: - master - staging + tags: + - "*" pull_request: - paths-ignore: - - '**.md' jobs: unit_tests: + name: "Unit Tests" runs-on: [self-hosted, load-test] steps: - name: Setup go 1.17 @@ -21,20 +22,14 @@ jobs: - name: Clone blobber uses: actions/checkout@v2 - # - name: Set GITHUB_ENV - # run: | - # echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - # echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - # echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - - name: Run tests run: | cd $GITHUB_WORKSPACE/code/go/0chain.net/ CGO_ENABLED=1 go test -tags bn256 -race -coverprofile=coverage.txt -covermode=atomic ./... - integration_test: + name: "Integration Tests" + needs: unit_tests runs-on: [ubuntu-20.04] timeout-minutes: 40 steps: @@ -47,23 +42,29 @@ jobs: - name: Clone Blobber uses: actions/checkout@v1 - - name: Build Blobber - run: | - docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 - ./docker.local/bin/build.base.sh - ./docker.local/bin/build.blobber.sh - - - name: Run Blobber on gRPC + - name: Install Postgres run: | ./docker.local/bin/blobber.init.setup.sh cd docker.local/blobber1 ../bin/blobber.start_github.sh docker ps - - name: Run Tests run: | + make local-build + sudo make local-run & + sleep 15 + running=0 + echo "checking blobber server status" + if [ $running -eq 0 ]; then + echo "checking blobber server status" + curl 127.0.0.1:5051 && $running=1 || sleep 2 + fi + + echo =========================[ run tests ]========================= + #sudo make integration-tests go17=$(which go) - sudo CGO_ENABLED=1 $go17 test -tags bn256 ./... -args integration + root=$(pwd) + sudo CGO_ENABLED=1 root=$root integration=1 $go17 test -tags bn256 ./... golangci: name: lint diff --git a/Makefile b/Makefile index 0ffa6ad29..65989554b 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,13 @@ ######################################################## ######################################################## ######################################################## - +UNAME_OS := $(shell uname -s) +UNAME_ARCH := $(shell uname -m) +ROOT := $(shell pwd) .PHONY: test test: - CGO_ENABLED=1 go test -tags bn256 ./...; + CGO_ENABLED=1 go test -tags bn256 ./... .PHONY: lint lint: @@ -22,10 +24,44 @@ lint: .PHONY: integration-tests integration-tests: - CGO_ENABLED=1 go test -tags bn256 ./... -args integration; + CGO_ENABLED=1 root=$(ROOT) integration=1 go test -p 1 -tags bn256 ./... + +.PHONY: local-init +local-init: + @echo "=========================[ init blobber ]=========================" + mkdir -p ./dev.local/data/blobber + #[ -d ./dev.local/data/blobber/config ] && rm -rf ./dev.local/data/blobber/config + cp -r ./config ./dev.local/data/blobber/ +ifeq ($(UNAME_OS),Darwin) + cd ./dev.local/data/blobber/config/ && find . -name "*.yaml" -exec sed -i '' "s/postgres/127.0.0.1/g" {} \; +else + cd ./dev.local/data/blobber/config/ && sed -i "s/postgres/127.0.0.1/g" ./0chain_blobber.yaml +endif + cd ./dev.local/data/blobber && [ -d files ] || mkdir files + cd ./dev.local/data/blobber && [ -d data ] || mkdir data + cd ./dev.local/data/blobber && [ -d log ] || mkdir log +.PHONY: local-build +local-build: local-init + @echo "=========================[ build blobber ]=========================" + cd ./code/go/0chain.net/blobber && CGO_ENABLED=1 go build -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=dev" -o ../../../../dev.local/data/blobber/blobber . +.PHONY: local-run +local-run: + @echo "=========================[ run blobber ]=========================" + cd ./dev.local/ && integration=1 ./data/blobber/blobber \ + --port 5051 \ + --grpc_port 31501 \ + --hostname 127.0.0.1 \ + --deployment_mode 0 \ + --keys_file ../docker.local/keys_config/b0bnode1_keys.txt \ + --files_dir ./data/blobber/files \ + --log_dir ./data/blobber/log \ + --db_dir ./data/blobber/data \ + --minio_file ../docker.local/keys_config/minio_config.txt \ + --config_dir ./data/blobber/config + ######################################################## ######################################################## @@ -68,8 +104,7 @@ BUF_INSTALL_FROM_SOURCE := false ### Everything below this line is meant to be static, i.e. only adjust the above variables. ### -UNAME_OS := $(shell uname -s) -UNAME_ARCH := $(shell uname -m) + # Buf will be cached to ~/.cache/buf-example. CACHE_BASE := $(HOME)/.cache/$(PROJECT) # This allows switching between i.e a Docker container and your local setup without overwriting. diff --git a/code/go/0chain.net/blobber/datastore.go b/code/go/0chain.net/blobber/datastore.go index dfda9f992..5402cfdba 100644 --- a/code/go/0chain.net/blobber/datastore.go +++ b/code/go/0chain.net/blobber/datastore.go @@ -18,7 +18,7 @@ func setupDatabase() error { } if err := datastore.GetStore().Open(); err == nil { - if i == 1 { // no more attempts + if i == 599 { // no more attempts logging.Logger.Error("Failed to connect to the database. Shutting the server down") return err } diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go index 598e1b5e8..92c50a2cb 100644 --- a/code/go/0chain.net/blobber/flags.go +++ b/code/go/0chain.net/blobber/flags.go @@ -3,19 +3,21 @@ package main import ( "flag" "fmt" + "os" ) var ( - deploymentMode int - keysFile string - minioFile string - filesDir string - metadataDB string - logDir string - httpPort int - hostname string - configDir string - grpcPort int + deploymentMode int + keysFile string + minioFile string + filesDir string + metadataDB string + logDir string + httpPort int + hostname string + configDir string + grpcPort int + isIntegrationTest bool ) func init() { @@ -52,5 +54,7 @@ func parseFlags() { if httpPort <= 0 { panic("Please specify --port which is the port on which requests are accepted") } + isIntegrationTest = os.Getenv("integration") == "1" + fmt.Print(" [OK]\n") } diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go index 962451b94..4da0db458 100644 --- a/code/go/0chain.net/blobber/grpc.go +++ b/code/go/0chain.net/blobber/grpc.go @@ -3,6 +3,7 @@ package main import ( + "errors" "fmt" "log" "net" @@ -17,18 +18,17 @@ import ( ) func startGRPCServer() { - fmt.Println("[10/11] starting grpc server [OK]") - - r := mux.NewRouter() common.ConfigRateLimits() + r := mux.NewRouter() initHandlers(r) + grpcServer := handler.NewGRPCServerWithMiddlewares(r) reflection.Register(grpcServer) if grpcPort <= 0 { logging.Logger.Error("grpc port missing") - return + panic(errors.New("grpc port missing")) } logging.Logger.Info("started grpc server on to grpc requests on port - " + strconv.Itoa(grpcPort)) @@ -36,6 +36,11 @@ func startGRPCServer() { lis, err := net.Listen("tcp", fmt.Sprintf(":%v", grpcPort)) if err != nil { log.Fatalf("failed to listen: %v", err) + panic(err) } - log.Fatal(grpcServer.Serve(lis)) + + fmt.Println("[10/11] starting grpc server [OK]") + go func() { + log.Fatal(grpcServer.Serve(lis)) + }() } diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index 6bbb932bb..cfbe85485 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -23,7 +23,6 @@ import ( var startTime time.Time func startHttpServer() { - fmt.Println("[11/11] start http server [OK]") mode := "main net" if config.Development() { @@ -38,9 +37,8 @@ func startHttpServer() { address := ":" + strconv.Itoa(httpPort) var server *http.Server - r := mux.NewRouter() - common.ConfigRateLimits() + r := mux.NewRouter() initHandlers(r) if config.Development() { @@ -65,7 +63,7 @@ func startHttpServer() { handler.HandleShutdown(common.GetRootContext()) logging.Logger.Info("Ready to listen to the requests") - + fmt.Println("[11/11] start http server [OK]") startTime = time.Now().UTC() log.Fatal(server.ListenAndServe()) diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 31c7fd36d..c6a2455bb 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -43,11 +43,8 @@ func main() { go setupOnChain() - go startGRPCServer() - - startHttpServer() - - + startGRPCServer() + startHttpServer() } diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index d602d727e..5c4bd29c7 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -15,17 +15,23 @@ import ( ) func setupOnChain() { + //wait http & grpc startup, and go to setup on chain + time.Sleep(1 * time.Second) fmt.Println("[9/11] connecting to chain ") const ATTEMPT_DELAY = 60 * 1 // setup wallet fmt.Print(" + connect to miners: ") - if err := handler.WalletRegister(); err != nil { - fmt.Println(err.Error() + "\n") - panic(err) + if isIntegrationTest { + fmt.Print(" [SKIP]\n") + } else { + if err := handler.WalletRegister(); err != nil { + fmt.Println(err.Error() + "\n") + panic(err) + } + fmt.Print(" [OK]\n") } - fmt.Print(" [OK]\n") // setup blobber (add or update) on the blockchain (multiple attempts) for i := 1; i <= 10; i++ { @@ -35,30 +41,37 @@ func setupOnChain() { fmt.Printf("\r + [%v/10]connect to sharders:", i) } - if err := registerBlobberOnChain(); err != nil { - if i == 10 { // no more attempts - panic(err) - } - fmt.Print("\n ", err.Error()+"\n") - - } else { - fmt.Print(" [OK]\n") + if isIntegrationTest { + fmt.Print(" [SKIP]\n") break - } - for n := 0; n < ATTEMPT_DELAY; n++ { - <-time.After(1 * time.Second) + } else { + + if err := registerBlobberOnChain(); err != nil { + if i == 10 { // no more attempts + panic(err) + } + fmt.Print("\n ", err.Error()+"\n") + + } else { + fmt.Print(" [OK]\n") + break + } + for n := 0; n < ATTEMPT_DELAY; n++ { + <-time.After(1 * time.Second) - fmt.Printf("\r - wait %v seconds to retry", ATTEMPT_DELAY-n) + fmt.Printf("\r - wait %v seconds to retry", ATTEMPT_DELAY-n) + } } } + if !isIntegrationTest { + go setupWorkers() - go setupWorkers() - - go keepAliveOnChain() + go keepAliveOnChain() - if config.Configuration.PriceInUSD { - go refreshPriceOnChain() + if config.Configuration.PriceInUSD { + go refreshPriceOnChain() + } } } @@ -86,11 +99,19 @@ func setupServerChain() error { chain.SetServerChain(serverChain) if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil { + if isIntegrationTest { + return nil + } + return err } if err := zcncore.SetWalletInfo(node.Self.GetWalletString(), false); err != nil { + if isIntegrationTest { + return nil + } return err } + fmt.Print(" [OK]\n") return nil } diff --git a/code/go/0chain.net/blobbercore/handler/download_integration_test.go b/code/go/0chain.net/blobbercore/handler/download_integration_test.go index 4f6bc7c53..3306be9ac 100644 --- a/code/go/0chain.net/blobbercore/handler/download_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/download_integration_test.go @@ -4,13 +4,14 @@ import ( "context" "encoding/hex" "encoding/json" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "io" "os" - "strings" + "path/filepath" "testing" "time" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" @@ -21,27 +22,26 @@ func TestBlobberGRPCService_DownloadFile(t *testing.T) { bClient, tdController := setupHandlerIntegrationTests(t) allocationTx := randString(32) - root, _ := os.Getwd() - path := strings.Split(root, `code`) + root := os.Getenv("root") - err := os.MkdirAll(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, os.ModePerm) + err := os.MkdirAll(filepath.Join(root, `dev.local/data/blobber/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`), os.ModePerm) if err != nil { t.Fatal(err) } defer func() { - err := os.RemoveAll(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon`) + err := os.RemoveAll(filepath.Join(root, `dev.local/data/blobber/files/files/exa/mpl/eId/objects/tmp/Mon`)) if err != nil { t.Fatal(err) } }() - f, err := os.Create(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) + f, err := os.Create(filepath.Join(root, `dev.local/data/blobber/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`)) if err != nil { t.Fatal(err) } defer f.Close() - file, err := os.Open(root + "/helper_integration_test.go") + file, err := os.Open(filepath.Join(root, "/code/go/0chain.net/blobbercore/handler/helper_integration_test.go")) if err != nil { t.Fatal(err) } diff --git a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go index d429b2e5c..ee438f396 100644 --- a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go @@ -43,11 +43,13 @@ func randString(n int) string { } func setupHandlerIntegrationTests(t *testing.T) (blobbergrpc.BlobberServiceClient, *TestDataController) { - args := make(map[string]bool) - for _, arg := range os.Args { - args[arg] = true - } - if !args["integration"] { + // args := make(map[string]bool) + // for _, arg := range os.Args { + // args[arg] = true + // } + + if os.Getenv("integration") != "1" { + // if !args["integration"] { t.Skip() } diff --git a/docker.local/b0docker-compose-github.yml b/docker.local/b0docker-compose-github.yml index 710414d59..303e5d8f1 100644 --- a/docker.local/b0docker-compose-github.yml +++ b/docker.local/b0docker-compose-github.yml @@ -26,26 +26,6 @@ services: links: - postgres:postgres - blobber: - image: blobber - environment: - - DOCKER= true - volumes: - - ../config:/blobber/config - - ./blobber${BLOBBER}/files:/blobber/files - - ./blobber${BLOBBER}/data:/blobber/data - - ./blobber${BLOBBER}/log:/blobber/log - - ./keys_config:/blobber/keysconfig - - ./blobber${BLOBBER}/data/tmp:/tmp - ports: - - 505${BLOBBER}:505${BLOBBER} - - 3150${BLOBBER}:3150${BLOBBER} - command: ./bin/blobber --port 505${BLOBBER} --grpc_port 3150${BLOBBER} --hostname 198.18.0.9${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data --minio_file keysconfig/minio_config.txt - networks: - default: - testnet0: - ipv4_address: 198.18.0.9${BLOBBER} - networks: default: driver: bridge diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index d658ffeb3..9e93e1247 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -3,7 +3,7 @@ LABEL zchain="blobber" ENV SRC_DIR=/0chain ENV GO111MODULE=on -#ENV GOPROXY=https://goproxy.cn,direct +# ENV GOPROXY=http://10.10.10.100:3080 # Download the dependencies: # Will be cached if we don't change mod/sum files From 643268433e61a53c7adb2ec94286c71995e8b848 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Nov 2021 05:51:24 +0800 Subject: [PATCH 053/161] chore(deps): bump gorm.io/gorm from 1.22.2 to 1.22.3 (#434) Bumps [gorm.io/gorm](https://github.com/go-gorm/gorm) from 1.22.2 to 1.22.3. - [Release notes](https://github.com/go-gorm/gorm/releases) - [Commits](https://github.com/go-gorm/gorm/compare/v1.22.2...v1.22.3) --- updated-dependencies: - dependency-name: gorm.io/gorm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0aee7565e..42c3a3435 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.2.1 - gorm.io/gorm v1.22.2 + gorm.io/gorm v1.22.3 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 9c1dbe1cc..89bc74ab1 100644 --- a/go.sum +++ b/go.sum @@ -1517,8 +1517,8 @@ gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.22.0/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.2 h1:1iKcvyJnR5bHydBhDqTwasOkoo6+o4Ms5cknSt6qP7I= -gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.3 h1:/JS6z+GStEQvJNW3t1FTwJwG/gZ+A7crFdRqtvG5ehA= +gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 523d8dc2426b85fc1292b5ef12b3682c2743aa70 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Nov 2021 06:06:52 +0800 Subject: [PATCH 054/161] chore(deps): bump gorm.io/driver/postgres from 1.2.1 to 1.2.2 (#430) Bumps [gorm.io/driver/postgres](https://github.com/go-gorm/postgres) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/go-gorm/postgres/releases) - [Commits](https://github.com/go-gorm/postgres/compare/v1.2.1...v1.2.2) --- updated-dependencies: - dependency-name: gorm.io/driver/postgres dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 42c3a3435..64ae6d985 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d - gorm.io/driver/postgres v1.2.1 + gorm.io/driver/postgres v1.2.2 gorm.io/gorm v1.22.3 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 89bc74ab1..a5d413418 100644 --- a/go.sum +++ b/go.sum @@ -1507,8 +1507,8 @@ gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d/go.mod h1:n2DTgk9at7cr/CWOT gorm.io/driver/mysql v0.3.1 h1:yvUT7Q0I3B9EHJ67NSp6cHbVwcdDHhVUsDAUiFFxRk0= gorm.io/driver/mysql v0.3.1/go.mod h1:A7H1JD9dKdcjeUTpTuWKEC+E1a74qzW7/zaXqKaTbfM= gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs= -gorm.io/driver/postgres v1.2.1 h1:JDQKnF7MC51dgL09Vbydc5kl83KkVDlcXfSPJ+xhh68= -gorm.io/driver/postgres v1.2.1/go.mod h1:SHRZhu+D0tLOHV5qbxZRUM6kBcf3jp/kxPz2mYMTsNY= +gorm.io/driver/postgres v1.2.2 h1:Ka9W6feOU+rPM9m007eYLMD4QoZuYGBnQ3Jp0faGSwg= +gorm.io/driver/postgres v1.2.2/go.mod h1:Ik3tK+a3FMp8ORZl29v4b3M0RsgXsaeMXh9s9eVMXco= gorm.io/driver/sqlite v1.0.8 h1:omllgSb7/eh9D6lGvLZOdU1ZElxdXuO3dn3Rk+dQxUE= gorm.io/driver/sqlite v1.0.8/go.mod h1:xkm8/CEmA3yc4zRd0pdCqm43BjO8Hm6avfTpxWb/7c4= gorm.io/driver/sqlserver v0.2.5 h1:o/MXpn9/BB68RXEEQzfhsSL382yEqUtdCiGIuCspmkY= @@ -1516,7 +1516,7 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.22.0/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gorm.io/gorm v1.22.3 h1:/JS6z+GStEQvJNW3t1FTwJwG/gZ+A7crFdRqtvG5ehA= gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= From 4a93e806934d499aa9f9c2e398918e37844d2e1a Mon Sep 17 00:00:00 2001 From: Artem Bogomaz Date: Tue, 16 Nov 2021 07:51:33 +0700 Subject: [PATCH 055/161] Expanding unit-tests && fixing issue with renaming dirs (#386) * :bug: expanding unit-tests && fixing issue with renaming dirs * :art: resolving unit tests * :art: lint fix * :hammer: more failing tests Co-authored-by: Lz --- .../allocation/renamefilechange.go | 6 + .../allocation/renamefilechange_test.go | 306 ++++++++++++++++++ .../0chain.net/blobbercore/handler/handler.go | 3 +- .../handler/object_operation_handler.go | 4 - .../0chain.net/blobbercore/reference/ref.go | 3 + 5 files changed, 316 insertions(+), 6 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index b4b7d9e97..bb445db05 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -68,6 +68,12 @@ func (rf *RenameFileChange) ProcessChange(ctx context.Context, change *Allocatio return nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") } } + + if len(dirRef.Children) == 0 { + Logger.Error("no files in root folder", zap.Any("change", rf)) + return nil, common.NewError("file_not_found", "No files in root folder") + } + idx := -1 for i, child := range dirRef.Children { if child.Path == rf.Path { diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go new file mode 100644 index 000000000..0a1ba18c9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -0,0 +1,306 @@ +package allocation + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + bconfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/config" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk/zboxcore/client" + zencryption "github.com/0chain/gosdk/zboxcore/encryption" + "github.com/0chain/gosdk/zcncore" + mocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/grpc/metadata" +) + +type MockFileBlockGetter struct { + filestore.IFileBlockGetter +} + +var mockFileBlock []byte + +func (MockFileBlockGetter) GetFileBlock( + fsStore *filestore.FileFSStore, + allocationID string, + fileData *filestore.FileInputData, + blockNum int64, + numBlocks int64) ([]byte, error) { + return []byte(mockFileBlock), nil +} + +func resetMockFileBlock() { + mockFileBlock = []byte("mock") +} + +var encscheme zencryption.EncryptionScheme + +func setupEncryptionScheme() { + encscheme = zencryption.NewEncryptionScheme() + mnemonic := client.GetClient().Mnemonic + if _, err := encscheme.Initialize(mnemonic); err != nil { + panic("initialize encscheme") + } + encscheme.InitForEncryption("filetype:audio") +} + +func setup(t *testing.T) { + // setup wallet + w, err := zcncrypto.NewBLS0ChainScheme().GenerateKeys() + if err != nil { + t.Fatal(err) + } + wBlob, err := json.Marshal(w) + if err != nil { + t.Fatal(err) + } + if err := zcncore.SetWalletInfo(string(wBlob), true); err != nil { + t.Fatal(err) + } + + // setup servers + sharderServ := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + }, + ), + ) + server := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + n := zcncore.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}} + blob, err := json.Marshal(n) + if err != nil { + t.Fatal(err) + } + + if _, err := w.Write(blob); err != nil { + t.Fatal(err) + } + }, + ), + ) + + if err := zcncore.InitZCNSDK(server.URL, "ed25519"); err != nil { + t.Fatal(err) + } +} + +func init() { + resetMockFileBlock() + common.ConfigRateLimits() + chain.SetServerChain(&chain.Chain{}) + config.Configuration.SignatureScheme = "bls0chain" + logging.Logger = zap.NewNop() + + dir, _ := os.Getwd() + if _, err := filestore.SetupFSStoreI(dir+"/tmp", MockFileBlockGetter{}); err != nil { + panic(err) + } + bconfig.Configuration.MaxFileSize = int64(1 << 30) +} + +func TestBlobberCore_RenameFile(t *testing.T) { + setup(t) + setupEncryptionScheme() + + sch := zcncrypto.NewBLS0ChainScheme() + sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" + _, err := sch.GenerateKeys() + if err != nil { + t.Fatal(err) + } + ts := time.Now().Add(time.Hour) + alloc := makeTestAllocation(common.Timestamp(ts.Unix())) + alloc.OwnerPublicKey = sch.GetPublicKey() + alloc.OwnerID = client.GetClientID() + + testCases := []struct { + name string + context metadata.MD + allocChange *AllocationChange + path string + newName string + allocRoot string + expectedMessage string + expectingError bool + setupDbMock func() + }{ + { + name: "Cant_find_file_object", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/old_dir", + newName: "/new_dir", + expectedMessage: "Invalid path. Could not find object tree", + expectingError: true, + setupDbMock: func() { + mocket.Catcher.Reset() + }, + }, + { + name: "Dirname_Change_Ok", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/old_dir", + newName: "/new_dir", + expectedMessage: "", + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."path" = $2 OR (path LIKE $3 AND allocation_id = $4)) AND "reference_objects"."deleted_at" IS NULL ORDER BY level, lookup_hash%!!(string=allocation id)!(string=/old_dir/%!)(MISSING)!(string=/old_dir)(EXTRA string=allocation id)` + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "reference_objects" WHERE`, + ).WithQuery(query). + WithReply( + []map[string]interface{}{{ + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/old_dir", + }}, + ) + + query = `SELECT * FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."parent_path" = $2 OR ("reference_objects"."allocation_id" = $3 AND "reference_objects"."parent_path" = $4) OR (parent_path = $5 AND allocation_id = $6)) AND "reference_objects"."deleted_at" IS NULL ORDER BY level, lookup_hash%!!(string=allocation id)!(string=)!(string=/)!(string=allocation id)!(string=/old_dir)(EXTRA string=allocation id)` + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "reference_objects" WHERE`, + ).WithQuery(query).WithReply( + []map[string]interface{}{{ + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "parent_path": ".", + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/old_dir", + "parent_path": "/", + }}, + ) + + mocket.Catcher.NewMock().WithQuery(`INSERT INTO "reference_objects"`). + WithID(1) + }, + }, + { + name: "Filename_Change_Ok", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "old_file.pdf", + newName: "new_file.pdf", + expectedMessage: "", + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."path" = $2 OR (path LIKE $3 AND allocation_id = $4)) AND "reference_objects"."deleted_at" IS NULL ORDER BY level, lookup_hash%!!(string=allocation id)!(string=old_file.pdf/%!)(MISSING)!(string=old_file.pdf)(EXTRA string=allocation id)` + mocket.Catcher.NewMock().OneTime().WithQuery(query). + WithReply( + []map[string]interface{}{{ + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "old_file.pdf", + }}, + ) + + query = `SELECT * FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."parent_path" = $2 OR ("reference_objects"."allocation_id" = $3 AND "reference_objects"."parent_path" = $4) OR (parent_path = $5 AND allocation_id = $6)) AND "reference_objects"."deleted_at" IS NULL ORDER BY level, lookup_hash%!!(string=allocation id)!(string=)!(string=.)!(string=allocation id)!(string=old_file.pdf)(EXTRA string=allocation id)` + mocket.Catcher.NewMock().OneTime().WithQuery(query).WithReply( + []map[string]interface{}{{ + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "parent_path": ".", + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "old_file.pdf", + "parent_path": "/", + }}, + ) + + query = `SELECT * FROM "reference_objects" WHERE "id" = $1 AND "reference_objects"."deleted_at" IS NULL ORDER BY "reference_objects"."id" LIMIT 1%!(EXTRA int64=1)` + mocket.Catcher.NewMock().OneTime().WithQuery(query). + WithReply( + []map[string]interface{}{{ + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "parent_path": ".", + }}, + ) + mocket.Catcher.NewMock().WithQuery(`INSERT INTO "reference_objects"`). + WithID(1) + }, + }, + } + + for _, tc := range testCases { + datastore.MocketTheStore(t, true) + tc.setupDbMock() + + ctx := context.TODO() + db := datastore.GetStore().GetDB().Begin() + ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + + change := &RenameFileChange{AllocationID: alloc.ID, Path: tc.path, NewName: tc.newName} + response, err := change.ProcessChange(ctx, tc.allocChange, tc.allocRoot) + + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + if tc.expectingError && strings.Contains(tc.expectedMessage, err.Error()) { + t.Fatal("expected error " + tc.expectedMessage) + break + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + require.EqualValues(t, len(response.Children), 1) + require.EqualValues(t, response.Children[0].Path, tc.newName) + } +} + +func makeTestAllocation(exp common.Timestamp) *Allocation { + allocID := "allocation id" + alloc := Allocation{ + Tx: "allocation tx", + ID: allocID, + Terms: []*Terms{ + { + ID: 1, + AllocationID: allocID, + }, + }, + Expiration: exp, + } + return &alloc +} diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 1f2fe485f..9f9b28737 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -41,12 +41,11 @@ func SetupHandlers(r *mux.Router) { //object operations r.HandleFunc("/v1/file/upload/{allocation}", common.ToJSONResponse(WithConnection(UploadHandler))) r.HandleFunc("/v1/file/download/{allocation}", common.ToByteStream(WithConnection(DownloadHandler))).Methods("POST") - r.HandleFunc("/v1/file/rename/{allocation}", common.ToJSONResponse(WithConnection(RenameHandler))) + r.HandleFunc("/v1/file/rename/{allocation}", common.ToJSONResponse(WithConnection(RenameHandler))).Methods("POST") r.HandleFunc("/v1/file/copy/{allocation}", common.ToJSONResponse(WithConnection(CopyHandler))) r.HandleFunc("/v1/file/attributes/{allocation}", common.ToJSONResponse(WithConnection(UpdateAttributesHandler))) r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("POST") r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("DELETE") - r.HandleFunc("/v1/dir/rename/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("POST") r.HandleFunc("/v1/connection/commit/{allocation}", common.ToJSONResponse(WithConnection(CommitHandler))) r.HandleFunc("/v1/file/commitmetatxn/{allocation}", common.ToJSONResponse(WithConnection(CommitMetaTxnHandler))) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index fbe9144ed..568ce8fb3 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -674,10 +674,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (interface{}, error) { - - if r.Method == "GET" { - return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") - } allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 2531d3960..adcaebf48 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -198,6 +198,9 @@ func GetSubDirsFromPath(p string) []string { parent = filepath.Clean(parent) subDirs := make([]string, 0) for len(cur) > 0 { + if cur == "." { + break + } subDirs = append([]string{cur}, subDirs...) parent, cur = filepath.Split(parent) parent = filepath.Clean(parent) From cd08727746b644ef0f03ddff18213291d19b6173 Mon Sep 17 00:00:00 2001 From: prince Date: Tue, 16 Nov 2021 17:05:03 +0530 Subject: [PATCH 056/161] issue 420: override "{}" when 'latest_redeemed_rm' is empty string. (#432) * issue 420: override "{}" when 'latest_redeemed_rm' is empty string. Signed-off-by: Pradip Parmar * mock query args added for update query. Signed-off-by: Pradip Parmar Co-authored-by: Lz --- .../blobbercore/handler/handler_test.go | 10 +++---- .../blobbercore/readmarker/entity.go | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 1569386e8..17b35901a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -1640,7 +1640,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() mock.ExpectExec(`(UPDATE "read_markers" SET)(.+)`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -1756,7 +1756,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). @@ -1888,7 +1888,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") @@ -2035,7 +2035,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") @@ -2182,7 +2182,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") diff --git a/code/go/0chain.net/blobbercore/readmarker/entity.go b/code/go/0chain.net/blobbercore/readmarker/entity.go index dcae8e536..3245c5eac 100644 --- a/code/go/0chain.net/blobbercore/readmarker/entity.go +++ b/code/go/0chain.net/blobbercore/readmarker/entity.go @@ -115,6 +115,7 @@ func SaveLatestReadMarker(ctx context.Context, rm *ReadMarker, isCreate bool) er rmEntity.LatestRM = rm rmEntity.RedeemRequired = true + rmEntity.LatestRedeemedRMBlob = datatypes.JSON("{}") if isCreate { return db.Create(rmEntity).Error @@ -128,16 +129,22 @@ func (rm *ReadMarkerEntity) Sync(ctx context.Context) (err error) { var db = datastore.GetStore().GetTransaction(ctx) - var rmUpdates = make(map[string]interface{}) - rmUpdates["latest_redeem_txn_id"] = "Synced from SC REST API" - rmUpdates["status_message"] = "sync" - rmUpdates["redeem_required"] = false - - var latestRMBytes []byte - if latestRMBytes, err = json.Marshal(rm.LatestRM); err != nil { - return common.NewErrorf("rme_sync", "marshaling latest RM: %v", err) - } - rmUpdates["latest_redeemed_rm"] = latestRMBytes + ////////////////////////////////////////////////////////////////////////////// + /////////////////////// Not using this map anywhere //////////////////////// + ////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////// + // var rmUpdates = make(map[string]interface{}) // + // rmUpdates["latest_redeem_txn_id"] = "Synced from SC REST API" // + // rmUpdates["status_message"] = "sync" // + // rmUpdates["redeem_required"] = false // + // // + // var latestRMBytes []byte // + // if latestRMBytes, err = json.Marshal(rm.LatestRM); err != nil { // + // return common.NewErrorf("rme_sync", "marshaling latest RM: %v", err) // + // } // + // rmUpdates["latest_redeemed_rm"] = latestRMBytes // + ////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////// // update local read pools cache from sharders var rps []*allocation.ReadPool @@ -184,6 +191,9 @@ func (rm *ReadMarkerEntity) UpdateStatus(ctx context.Context, return common.NewErrorf("rme_update_status", "marshaling latest RM: %v", err) } + if len(latestRMBytes) == 0 || string(latestRMBytes) == "" { + latestRMBytes = []byte("{}") + } rmUpdates["latest_redeemed_rm"] = latestRMBytes // the saving looses the numBlocks information From 3dd0b3d5bfe9fd8954b030da532ee544c2640afa Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 16 Nov 2021 11:53:51 +0000 Subject: [PATCH 057/161] Trigger system tests from blobber PR (#427) * Trigger system tests from blobber PR * Add skip tests option * change text * change if conditions * add concurrency * add concurrency * test commit * add configurable system tests branch * add configurable system tests branch Co-authored-by: Lz --- .github/workflows/system_tests.yml | 85 +++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 022c5c8f4..d4c40c3ed 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -1,8 +1,16 @@ -name: "0Chain System Tests" +name: "0Chain System Tests" + +concurrency: + group: "${{ github.ref }}" + cancel-in-progress: true on: workflow_dispatch: inputs: + system_tests_branch: + description: 'system_tests branch. Containing the tests you wish to run' + default: 'master' + required: true zbox_cli_branch: description: '0Box CLI (branch or commit SHA) which the tests will use' default: 'staging' @@ -35,11 +43,21 @@ on: description: '0block DOCKER IMAGE to deploy' default: 'staging' required: true + skip_tests: + description: '(LAST RESORT ONLY) Skip system tests. This will allow a PR to merge without requiring a green test run. *By using you certify that the code being merged is not causing system tests to fail*' + default: 'FALSE' + required: true jobs: system-tests: runs-on: [ tests-suite ] + timeout-minutes: 40 steps: + - uses: jwalton/gh-find-current-pr@v1 + id: findPr + with: + github-token: ${{ github.token }} + - name: Extract branch name shell: bash run: | @@ -48,3 +66,68 @@ jobs: echo "##[set-output name=pr_number;]$(echo $pull_number)" id: extract_branch + - name: Set PR Status Pending + uses: niteoweb/pull_request_status_action@v1.0.0 + if: steps.findPr.outputs.number + with: + pr_number: ${{ steps.findPr.outputs.pr }} + description: "System tests running" + state: "pending" + repository: ${{ github.repository }} + context: ${{ github.workflow }} + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ github.token }} + + - uses: stewartie4/trigger-workflow-and-wait@master + name: "Run System Tests" + if: ${{ github.event.inputs.skip_tests == 'FALSE' }} + with: + owner: 0chain + repo: system_test + github_token: ${{ secrets.SVC_ACCOUNT_SECRET }} + workflow_file_name: ci.yml + ref: github.event.inputs.system_tests_branch + wait_interval: 60 + inputs: > + { + "test_run_description" : "Triggered by [${{ github.event.repository.name }}] repo, branch [${{ steps.extract_branch.outputs.branch }}], URL [https://github.com/0chain/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}]", + "zbox_cli_branch" : "${{github.event.inputs.zbox_cli_branch}}", + "zwallet_cli_branch" : "${{github.event.inputs.zwallet_cli_branch}}", + "miner_image": "${{github.event.inputs.miner_image}}", + "sharder_image": "${{github.event.inputs.sharder_image}}", + "blobber_image": "${{github.event.inputs.blobber_image}}", + "validator_image": "${{github.event.inputs.validator_image}}", + "zbox_image": "${{github.event.inputs.zbox_image}}", + "zblock_image": "${{github.event.inputs.zblock_image}}" + } + propagate_failure: true + trigger_workflow: true + wait_workflow: true + github_user: 'service-0chain' + + - name: Set PR Status Success + if: ${{ success() && steps.findPr.outputs.number }} + uses: niteoweb/pull_request_status_action@v1.0.0 + with: + pr_number: ${{ steps.findPr.outputs.pr }} + description: "System tests success" + state: ${{ job.status }} + repository: ${{ github.repository }} + context: ${{ github.workflow }} + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Set PR Status Failed + if: ${{ failure() && steps.findPr.outputs.number }} + uses: niteoweb/pull_request_status_action@v1.0.0 + with: + pr_number: ${{ steps.findPr.outputs.pr }} + description: "System tests failure" + state: ${{ job.status }} + repository: ${{ github.repository }} + context: ${{ github.workflow }} + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ github.token }} From 8fa8e7292aff1e8ac82a27a0f198fd883156401c Mon Sep 17 00:00:00 2001 From: Artem Bogomaz Date: Wed, 17 Nov 2021 15:06:12 +0700 Subject: [PATCH 058/161] :bug: Reject rename if file already exists (#390) * :bug: reject rename if file already exists * :hammer: make it more clean * Update code/go/0chain.net/blobbercore/allocation/renamefilechange.go * :hammer: removing unnecesery code Co-authored-by: Kush Trivedi <44091822+kushthedude@users.noreply.github.com> Co-authored-by: Lz --- .../allocation/renamefilechange.go | 6 ++++++ .../blobbercore/reference/referencepath.go | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index bb445db05..55e05a572 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -25,10 +25,16 @@ func (rf *RenameFileChange) DeleteTempFile() error { } func (rf *RenameFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { + isFilePresent, _ := reference.PathExists(ctx, rf.AllocationID, rf.NewName) + if isFilePresent { + return nil, common.NewError("invalid_reference_path", "file already exists") + } + affectedRef, err := reference.GetObjectTree(ctx, rf.AllocationID, rf.Path) if err != nil { return nil, err } + path, _ := filepath.Split(affectedRef.Path) path = filepath.Clean(path) affectedRef.Name = rf.NewName diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 396465e5d..2d3e84b5b 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -78,6 +78,26 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths [ return &refs[0], nil } +func PathExists(ctx context.Context, allocationID string, path string) (bool, error) { + path = filepath.Clean(path) + + if path == "." || path == "/" { + return false, nil + } + + var refs []Ref + db := datastore.GetStore().GetTransaction(ctx) + + ref := Ref{Path: path, AllocationID: allocationID} + err := db.Where(ref).Find(&refs).Error + + if err != nil || len(refs) == 0 { + return false, err + } + + return true, nil +} + func GetObjectTree(ctx context.Context, allocationID string, path string) (*Ref, error) { path = filepath.Clean(path) var refs []Ref From 7de518e56c36202a7f76205b8c0bf581316b583c Mon Sep 17 00:00:00 2001 From: Andika Demas Riyandi Date: Wed, 17 Nov 2021 19:34:20 +0700 Subject: [PATCH 059/161] fix zbox list-all wrong return, and make error format clearer (#435) Co-authored-by: Lz --- code/go/0chain.net/blobbercore/allocation/protocol.go | 9 ++++----- .../go/0chain.net/blobbercore/handler/storage_handler.go | 7 +++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 25d03d9bb..4d0dfc327 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -12,7 +12,6 @@ import ( . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - "gorm.io/gorm" ) @@ -61,7 +60,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, First(a).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, err // unexpected DB error + return nil, common.NewError("unexpected database error", err.Error()) // unexpected DB error } if err == nil { @@ -71,7 +70,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, Where("allocation_id = ?", a.ID). Find(&terms).Error if err != nil { - return // unexpected DB error + return nil, common.NewError("unexpected database error", err.Error()) // unexpected DB error } a.Terms = terms // set field return // found in DB @@ -94,7 +93,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, Where("id = ?", sa.ID). First(a).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, err // unexpected + return nil, common.NewError("unexpected database error", err.Error()) // unexpected } isExist = (a.ID != "") @@ -142,7 +141,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, } if readonly { - return + return a, nil } Logger.Info("Saving the allocation to DB") diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index bda20ebb4..468216f60 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -157,12 +157,12 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in } var ( - isOwner = clientID == alloc.OwnerID - isRepairer = clientID == alloc.RepairerID + isOwner = clientID == alloc.OwnerID + isRepairer = clientID == alloc.RepairerID isCollaborator = reference.IsACollaborator(ctx, fileref.ID, clientID) ) - if isOwner || isCollaborator{ + if isOwner || isCollaborator { publicKey := alloc.OwnerPublicKey if isCollaborator { publicKey = ctx.Value(constants.ContextKeyClientKey).(string) @@ -417,7 +417,6 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* clientID := ctx.Value(constants.ContextKeyClient).(string) allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, true) - if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } From 6f02cddbc23b2ba34e7058f509c5e5cda23b2efb Mon Sep 17 00:00:00 2001 From: Andika Demas Riyandi Date: Thu, 18 Nov 2021 13:48:06 +0700 Subject: [PATCH 060/161] Update/gosdk (#436) * fix zbox list-all wrong return, and make error format clearer * upgrade gosdk Co-authored-by: Lz --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 64ae6d985..f3b95657b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1 + github.com/0chain/gosdk v1.3.1-0.20211112032529-5a968d2a4823 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index a5d413418..a773097d4 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1 h1:7woeGwzIoqGVfVIcEQ2R0S0Ebi9dyBIRCd0zHIdxEuM= -github.com/0chain/gosdk v1.3.1-0.20211110141100-0523404d2ee1/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/0chain/gosdk v1.3.1-0.20211112032529-5a968d2a4823 h1:5E2yZYuacKSmWEdgQklpOqAk/lw41yrPomq6y/7B0cU= +github.com/0chain/gosdk v1.3.1-0.20211112032529-5a968d2a4823/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= From 97aaef4457d48cb90f5799138d67e5659eaa2a56 Mon Sep 17 00:00:00 2001 From: Andika Demas Riyandi Date: Fri, 19 Nov 2021 17:30:49 +0700 Subject: [PATCH 061/161] fix wrong error output and remove readonly check for storage hander (#438) * fix zbox list-all wrong return, and make error format clearer * remove readOnly check, and fix wrong error return * fix(list):#426 fixed invlaid error for zbox list-all Co-authored-by: Lz --- .../blobbercore/allocation/protocol.go | 8 ++--- .../0chain.net/blobbercore/handler/handler.go | 2 +- .../blobbercore/handler/storage_handler.go | 34 +++++++++++++++---- go.mod | 2 +- go.sum | 4 +-- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 4d0dfc327..e4c358df9 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -60,7 +60,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, First(a).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, common.NewError("unexpected database error", err.Error()) // unexpected DB error + return nil, common.NewError("bad_db_operation", err.Error()) // unexpected DB error } if err == nil { @@ -70,7 +70,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, Where("allocation_id = ?", a.ID). Find(&terms).Error if err != nil { - return nil, common.NewError("unexpected database error", err.Error()) // unexpected DB error + return nil, common.NewError("bad_db_operation", err.Error()) // unexpected DB error } a.Terms = terms // set field return // found in DB @@ -93,7 +93,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, Where("id = ?", sa.ID). First(a).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, common.NewError("unexpected database error", err.Error()) // unexpected + return nil, common.NewError("bad_db_operation", err.Error()) // unexpected } isExist = (a.ID != "") @@ -168,7 +168,7 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, } } - return + return a, nil } // read/write pool stat for an {allocation -> blobber} diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 9f9b28737..538a4cdfa 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -56,7 +56,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/allocation", common.ToJSONResponse(WithConnection(AllocationHandler))) r.HandleFunc("/v1/file/meta/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(FileMetaHandler))) r.HandleFunc("/v1/file/stats/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(FileStatsHandler))) - r.HandleFunc("/v1/file/list/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ListHandler))) + r.HandleFunc("/v1/file/list/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ListHandler))).Methods("GET") r.HandleFunc("/v1/file/objectpath/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler))) r.HandleFunc("/v1/file/referencepath/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler))) r.HandleFunc("/v1/file/objecttree/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler))) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 468216f60..ace2eb2ce 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -3,11 +3,13 @@ package handler import ( "context" "encoding/json" + "errors" "net/http" "strconv" "strings" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" @@ -411,12 +413,9 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*blobberhttp.ListResult, error) { - if r.Method == "POST" { - return nil, common.NewError("invalid_method", "Invalid method used. Use GET instead") - } clientID := ctx.Value(constants.ContextKeyClient).(string) allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) - allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, true) + allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } @@ -426,7 +425,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - pathHash, err := pathHashFromReq(r, allocationID) + pathHash, path, err := getPathHash(r, allocationID) if err != nil { return nil, err } @@ -435,8 +434,16 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* fileref, err := reference.GetReferenceFromLookupHash(ctx, allocationID, pathHash) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid path. "+err.Error()) + if errors.Is(err, gorm.ErrRecordNotFound) { + // `/` always is valid even it doesn't exists in db. so ignore RecordNotFound error + if path != "/" { + return nil, common.NewError("invalid_parameters", "Invalid path "+err.Error()) + } + } else { + return nil, common.NewError("bad_db_operation", err.Error()) + } } + authTokenString := r.FormValue("auth_token") if clientID != allocationObj.OwnerID || len(authTokenString) > 0 { authTicketVerified, err := fsh.verifyAuthTicket(ctx, r.FormValue("auth_token"), allocationObj, fileref, clientID) @@ -888,3 +895,18 @@ func pathHashFromReq(r *http.Request, allocationID string) (string, error) { return pathHash, nil } + +func getPathHash(r *http.Request, allocationID string) (string, string, error) { + var ( + pathHash = r.FormValue("path_hash") + path = r.FormValue("path") + ) + if len(pathHash) == 0 { + if len(path) == 0 { + return "", "", common.NewError("invalid_parameters", "Invalid path") + } + pathHash = reference.GetReferenceLookup(allocationID, path) + } + + return pathHash, path, nil +} diff --git a/go.mod b/go.mod index f3b95657b..c25e7c0a9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.3.1-0.20211112032529-5a968d2a4823 + github.com/0chain/gosdk v1.3.1-0.20211119021259-7c9c46917132 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index a773097d4..197b14017 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.3.1-0.20211112032529-5a968d2a4823 h1:5E2yZYuacKSmWEdgQklpOqAk/lw41yrPomq6y/7B0cU= -github.com/0chain/gosdk v1.3.1-0.20211112032529-5a968d2a4823/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/0chain/gosdk v1.3.1-0.20211119021259-7c9c46917132 h1:7dk7FJY0sE79BRz7aH2o+VafMe3GoB2wIXBCpsJfp8o= +github.com/0chain/gosdk v1.3.1-0.20211119021259-7c9c46917132/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= From dc5c2dcaaab8928f7b0c619fb8febc54ae6d0e1f Mon Sep 17 00:00:00 2001 From: stewartie4 Date: Sun, 21 Nov 2021 12:35:07 +0000 Subject: [PATCH 062/161] fix substitution bug --- .github/workflows/system_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index d4c40c3ed..ae07fd1af 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -87,7 +87,7 @@ jobs: repo: system_test github_token: ${{ secrets.SVC_ACCOUNT_SECRET }} workflow_file_name: ci.yml - ref: github.event.inputs.system_tests_branch + ref: ${{ github.event.inputs.system_tests_branch }} wait_interval: 60 inputs: > { From 40fa394c3e4b0b891da0c13aadd7855a1d63f3db Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 23 Nov 2021 21:57:40 +0800 Subject: [PATCH 063/161] fix(devops):fixed docker image tag --- .../build-&-publish-docker-image.yml | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 67c154362..f73db0105 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -46,9 +46,8 @@ jobs: steps: - name: Set GITHUB_ENV run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 @@ -84,9 +83,8 @@ jobs: steps: - name: Set GITHUB_ENV run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Download blobber uses: actions/download-artifact@v2 @@ -120,9 +118,8 @@ jobs: steps: - name: Set GITHUB_ENV run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 @@ -160,9 +157,8 @@ jobs: steps: - name: Set GITHUB_ENV run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Download validator uses: actions/download-artifact@v2 From 7d4af78d867ea54b44e48aed2104df2038df2d85 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 23 Nov 2021 22:05:26 +0800 Subject: [PATCH 064/161] fix(devops):fixed docker image tag --- .github/workflows/build-&-publish-docker-image.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index f73db0105..65c6ed0fb 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -22,7 +22,7 @@ jobs: - name: Set GITHUB_ENV run: | echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 @@ -47,7 +47,7 @@ jobs: - name: Set GITHUB_ENV run: | echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 @@ -84,7 +84,7 @@ jobs: - name: Set GITHUB_ENV run: | echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Download blobber uses: actions/download-artifact@v2 @@ -119,7 +119,7 @@ jobs: - name: Set GITHUB_ENV run: | echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 @@ -158,7 +158,7 @@ jobs: - name: Set GITHUB_ENV run: | echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/}) | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - name: Download validator uses: actions/download-artifact@v2 From 8cea1cf99986a78cdc285e07c307e7585a274997 Mon Sep 17 00:00:00 2001 From: Rezoan Tamal <36768796+iamrz1@users.noreply.github.com> Date: Mon, 29 Nov 2021 08:52:33 +0600 Subject: [PATCH 065/161] [Feature] Improved stats page (#431) * Improved stats (infra and db info, failed challenge list) * Added requested changes from PR review * Removed logs and merged dbstats function * fix(stats):make code more clean Co-authored-by: Lz --- code/go/0chain.net/blobber/http.go | 31 +- code/go/0chain.net/blobber/worker.go | 20 +- code/go/0chain.net/blobber/zcn.go | 21 +- .../0chain.net/blobbercore/handler/handler.go | 89 ++- .../0chain.net/blobbercore/handler/health.go | 39 ++ .../blobbercore/handler/protocol.go | 23 +- .../blobbercore/stats/allocationstats.go | 3 +- .../blobbercore/stats/blobberstats.go | 99 ++- .../blobbercore/stats/challengestats.go | 52 ++ .../0chain.net/blobbercore/stats/dbstats.go | 11 + .../0chain.net/blobbercore/stats/filestats.go | 2 - .../0chain.net/blobbercore/stats/handler.go | 596 ++++++++++++------ .../blobbercore/stats/infrastats.go | 21 + .../blobbercore/stats/pagination.go | 108 ++++ code/go/0chain.net/core/common/time.go | 4 +- 15 files changed, 826 insertions(+), 293 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/health.go create mode 100644 code/go/0chain.net/blobbercore/stats/challengestats.go create mode 100644 code/go/0chain.net/blobbercore/stats/dbstats.go create mode 100644 code/go/0chain.net/blobbercore/stats/infrastats.go create mode 100644 code/go/0chain.net/blobbercore/stats/pagination.go diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index cfbe85485..0886dbdd9 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -10,18 +10,12 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" - "github.com/0chain/blobber/code/go/0chain.net/core/build" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/gosdk/zcncore" "github.com/gorilla/mux" "go.uber.org/zap" ) -var startTime time.Time - func startHttpServer() { mode := "main net" @@ -64,33 +58,12 @@ func startHttpServer() { logging.Logger.Info("Ready to listen to the requests") fmt.Println("[11/11] start http server [OK]") - startTime = time.Now().UTC() log.Fatal(server.ListenAndServe()) } func initHandlers(r *mux.Router) { - r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - mc := chain.GetServerChain() - - fmt.Fprintf(w, "
Running since %v ...\n", startTime) - fmt.Fprintf(w, "
Working on the chain: %v
\n", mc.ID) - fmt.Fprintf(w, - "
I am a blobber with
  • id:%v
  • public_key:%v
  • build_tag:%v
\n", - node.Self.ID, node.Self.PublicKey, build.BuildTag, - ) - - fmt.Fprintf(w, "
Miners ...\n") - network := zcncore.GetNetwork() - for _, miner := range network.Miners { - fmt.Fprintf(w, "%v\n", miner) - } - - fmt.Fprintf(w, "
Sharders ...\n") - for _, sharder := range network.Sharders { - fmt.Fprintf(w, "%v\n", sharder) - } - }) - + handler.StartTime = time.Now().UTC() + r.HandleFunc("/", handler.HomepageHandler) handler.SetupHandlers(r) } diff --git a/code/go/0chain.net/blobber/worker.go b/code/go/0chain.net/blobber/worker.go index 0a0cef16b..a4fb3a8d4 100644 --- a/code/go/0chain.net/blobber/worker.go +++ b/code/go/0chain.net/blobber/worker.go @@ -10,7 +10,9 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/spf13/viper" + "go.uber.org/zap" ) func setupWorkers() { @@ -28,18 +30,28 @@ func refreshPriceOnChain() { for { time.Sleep(REPEAT_DELAY * time.Second) if err := registerBlobberOnChain(); err != nil { - continue // pass // required by linting + logging.Logger.Error("refresh price on chain ", zap.Error(err)) } } } -func keepAliveOnChain() { +func healthCheckOnChain() { const REPEAT_DELAY = 60 * 15 // 15 minutes for { time.Sleep(REPEAT_DELAY * time.Second) - if err := healthCheckOnChain(); err != nil { - continue // pass // required by linting + txnHash, err := handler.BlobberHealthCheck() + if err != nil { + handler.SetBlobberHealthError(err) + } else { + t, err := handler.TransactionVerify(txnHash) + if err != nil { + logging.Logger.Error("Failed to verify blobber health check", zap.Any("err", err), zap.String("txn.Hash", txnHash)) + } else { + logging.Logger.Info("Verified blobber health check", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) + } + + handler.SetBlobberHealthError(err) } } } diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index 5c4bd29c7..3c6cb4edd 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -67,7 +67,7 @@ func setupOnChain() { if !isIntegrationTest { go setupWorkers() - go keepAliveOnChain() + go healthCheckOnChain() if config.Configuration.PriceInUSD { go refreshPriceOnChain() @@ -115,22 +115,3 @@ func setupServerChain() error { fmt.Print(" [OK]\n") return nil } - -func healthCheckOnChain() error { - txnHash, err := handler.BlobberHealthCheck(common.GetRootContext()) - if err != nil { - if err == handler.ErrBlobberHasRemoved { - return nil - } else { - return err - } - } - - if t, err := handler.TransactionVerify(txnHash); err != nil { - logging.Logger.Error("Failed to verify blobber health check", zap.Any("err", err), zap.String("txn.Hash", txnHash)) - } else { - logging.Logger.Info("Verified blobber health check", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) - } - - return err -} diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 538a4cdfa..5ca61d8bf 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -1,3 +1,4 @@ +//go:build !integration_tests // +build !integration_tests package handler @@ -6,28 +7,32 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "os" "runtime/pprof" "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/gosdk/zboxcore/fileref" - "gorm.io/gorm" - - "go.uber.org/zap" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" + "github.com/0chain/blobber/code/go/0chain.net/core/build" + "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk/zcncore" "github.com/gorilla/mux" + "go.uber.org/zap" + "gorm.io/gorm" ) var storageHandler StorageHandler +var StartTime time.Time func GetMetaDataStore() datastore.Store { return datastore.GetStore() @@ -64,7 +69,7 @@ func SetupHandlers(r *mux.Router) { //admin related r.HandleFunc("/_debug", common.ToJSONResponse(DumpGoRoutines)) r.HandleFunc("/_config", common.ToJSONResponse(GetConfig)) - r.HandleFunc("/_stats", stats.StatsHandler) + r.HandleFunc("/_stats", StatsHandler) r.HandleFunc("/_statsJSON", common.ToJSONResponse(stats.StatsJSONHandler)) r.HandleFunc("/_cleanupdisk", common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler))) r.HandleFunc("/getstats", common.ToJSONResponse(stats.GetStatsHandler)) @@ -127,6 +132,31 @@ func setupHandlerContext(ctx context.Context, r *http.Request) context.Context { return ctx } +func HomepageHandler(w http.ResponseWriter, r *http.Request) { + mc := chain.GetServerChain() + + fmt.Fprintf(w, "
Working on the chain: %v
\n", mc.ID) + fmt.Fprintf(w, + "
I am a blobber with
  • id:%v
  • public_key:%v
  • build_tag:%v
\n", + node.Self.ID, node.Self.PublicKey, build.BuildTag, + ) + + fmt.Fprintf(w, "
Miners ...\n") + network := zcncore.GetNetwork() + for _, miner := range network.Miners { + fmt.Fprintf(w, "%v\n", miner) + } + fmt.Fprintf(w, "
\n") + fmt.Fprintf(w, "
Sharders ...\n") + for _, sharder := range network.Sharders { + fmt.Fprintf(w, "%v\n", sharder) + } + fmt.Fprintf(w, "
\n") + fmt.Fprintf(w, "
") + fmt.Fprintf(w, "
Running since %v (Total elapsed time: %v)
\n", StartTime.Format(common.DateTimeFormat), time.Since(StartTime)) + fmt.Fprintf(w, "
") +} + func AllocationHandler(ctx context.Context, r *http.Request) (interface{}, error) { ctx = setupHandlerContext(ctx, r) @@ -328,6 +358,21 @@ func CalculateHashHandler(ctx context.Context, r *http.Request) (interface{}, er return response, nil } +func StatsHandler(w http.ResponseWriter, r *http.Request) { + HTMLHeader(w, "Blobber Diagnostics") + PrintCSS(w) + HomepageHandler(w, r) + err := GetBlobberHealthError() + if err != nil { + r.Header.Set(stats.HealthDataKey.String(), "✗") + } else { + r.Header.Set(stats.HealthDataKey.String(), "✔") + } + + stats.StatsHandler(w, r) + HTMLFooter(w) +} + //nolint:gosimple // need more time to verify func HandleShutdown(ctx context.Context) { go func() { @@ -489,3 +534,31 @@ func MarketPlaceShareInfoHandler(ctx context.Context, r *http.Request) (interfac return nil, errors.New("invalid request method, only POST is allowed") } + +//PrintCSS - print the common css elements +func PrintCSS(w http.ResponseWriter) { + fmt.Fprintf(w, "") +} + +func HTMLHeader(w http.ResponseWriter, title string) { + fmt.Fprintf(w, "%s", title) +} +func HTMLFooter(w http.ResponseWriter) { + fmt.Fprintf(w, "") +} diff --git a/code/go/0chain.net/blobbercore/handler/health.go b/code/go/0chain.net/blobbercore/handler/health.go new file mode 100644 index 000000000..b57cad5bc --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/health.go @@ -0,0 +1,39 @@ +package handler + +import ( + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "go.uber.org/zap" +) + +var blobberHealthCheckErr error + +func SetBlobberHealthError(err error) { + blobberHealthCheckErr = err +} + +func GetBlobberHealthError() error { + return blobberHealthCheckErr +} + +func BlobberHealthCheck() (string, error) { + if config.Configuration.Capacity == 0 { + return "", ErrBlobberHasRemoved + } + + txn, err := transaction.NewTransactionEntity() + if err != nil { + return "", err + } + + err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, + transaction.BLOBBER_HEALTH_CHECK, "", 0) + if err != nil { + logging.Logger.Info("Failed to health check on the blockchain", + zap.String("err:", err.Error())) + return "", err + } + + return txn.Hash, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go index 6c9a38f44..73e4cbe6c 100644 --- a/code/go/0chain.net/blobbercore/handler/protocol.go +++ b/code/go/0chain.net/blobbercore/handler/protocol.go @@ -116,7 +116,7 @@ func BlobberAdd(ctx context.Context) (string, error) { err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.ADD_BLOBBER_SC_NAME, string(snBytes), 0) if err != nil { - Logger.Info("Failed to set blobber on the blockchain", + Logger.Error("Failed to set blobber on the blockchain", zap.String("err:", err.Error())) return "", err } @@ -130,27 +130,6 @@ func BlobberAdd(ctx context.Context) (string, error) { // transactions. var ErrBlobberHasRemoved = errors.New("blobber has removed") -func BlobberHealthCheck(ctx context.Context) (string, error) { - if config.Configuration.Capacity == 0 { - return "", ErrBlobberHasRemoved - } - - txn, err := transaction.NewTransactionEntity() - if err != nil { - return "", err - } - - err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, - transaction.BLOBBER_HEALTH_CHECK, "", 0) - if err != nil { - Logger.Info("Failed to health check on the blockchain", - zap.String("err:", err.Error())) - return "", err - } - - return txn.Hash, nil -} - func TransactionVerify(txnHash string) (t *transaction.Transaction, err error) { for i := 0; i < util.MAX_RETRIES; i++ { diff --git a/code/go/0chain.net/blobbercore/stats/allocationstats.go b/code/go/0chain.net/blobbercore/stats/allocationstats.go index 55ddafaab..a680d3245 100644 --- a/code/go/0chain.net/blobbercore/stats/allocationstats.go +++ b/code/go/0chain.net/blobbercore/stats/allocationstats.go @@ -1,9 +1,8 @@ package stats import ( - "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "time" ) // Timestamp that implements standard fmt.Stringer interface. diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index 6fe5b15a9..73ef155d2 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -4,18 +4,16 @@ import ( "context" "database/sql" "encoding/json" + "runtime" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" - - "github.com/0chain/blobber/code/go/0chain.net/core/node" - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/node" "go.uber.org/zap" - "gorm.io/datatypes" ) @@ -60,9 +58,9 @@ type Stats struct { var LastMinioScan time.Time type MinioStats struct { - CloudFilesSize int64 `json:"cloud_files_size"` - CloudTotalFiles int `json:"cloud_total_files"` - LastMinioScan string `json:"last_minio_scan"` + CloudFilesSize int64 `json:"cloud_files_size"` + CloudTotalFiles int `json:"cloud_total_files"` + LastMinioScan time.Time `json:"last_minio_scan"` } type Duration int64 @@ -74,9 +72,14 @@ func (d Duration) String() string { type BlobberStats struct { Stats MinioStats - NumAllocation int64 `json:"num_of_allocations"` - ClientID string `json:"-"` - PublicKey string `json:"-"` + NumAllocation int64 `json:"num_of_allocations"` + ClientID string `json:"-"` + PublicKey string `json:"-"` + InfraStats InfraStats `json:"-"` + DBStats *DBStats `json:"-"` + FailedChallengeList []ChallengeEntity `json:"-"` + FailedChallengePagination Pagination `json:"-"` + AllocationListPagination Pagination `json:"-"` // configurations Capacity int64 `json:"capacity"` @@ -103,6 +106,9 @@ func LoadBlobberStats(ctx context.Context) *BlobberStats { fs := &BlobberStats{} fs.loadBasicStats(ctx) fs.loadDetailedStats(ctx) + fs.loadInfraStats(ctx) + fs.loadDBStats() + fs.loadFailedChallengeList(ctx) return fs } @@ -173,6 +179,58 @@ func (bs *BlobberStats) loadDetailedStats(ctx context.Context) { } } +func (bs *BlobberStats) loadInfraStats(ctx context.Context) { + healthIn := ctx.Value(HealthDataKey) + if healthIn == nil { + Logger.Error("loadInfraStats err where health value nil") + return + } + health := healthIn.(string) + memstats := runtime.MemStats{} + runtime.ReadMemStats(&memstats) + bs.InfraStats = InfraStats{ + CPUs: runtime.NumCPU(), + NumberOfGoroutines: runtime.NumGoroutine(), + HeapSys: int64(memstats.HeapSys), + HeapAlloc: int64(memstats.HeapAlloc), + ActiveOnChain: health, + } +} + +func (bs *BlobberStats) loadDBStats() { + bs.DBStats = &DBStats{Status: "✗"} + + db := datastore.GetStore().GetDB() + sqldb, err := db.DB() + if err != nil { + return + } + + dbStats := sqldb.Stats() + + bs.DBStats.Status = "✔" + bs.DBStats.DBStats = dbStats +} + +func (bs *BlobberStats) loadFailedChallengeList(ctx context.Context) { + fcrdI := ctx.Value(FailedChallengeRequestDataKey) + if fcrdI == nil { + Logger.Error("fcrd should not be nil") + return + } + fcrd := fcrdI.(RequestData) + + fcs, count, err := getAllFailedChallenges(fcrd.Offset, fcrd.Limit) + if err != nil { + Logger.Error("", zap.Any("err", err)) + return + } + bs.FailedChallengeList = fcs + + pagination := GeneratePagination(fcrd.Page, fcrd.Limit, fcrd.Offset, count) + bs.FailedChallengePagination = *pagination +} + func (bs *BlobberStats) loadStats(ctx context.Context) { const sel = ` @@ -232,7 +290,7 @@ func (bs *BlobberStats) loadMinioStats(ctx context.Context) { return } - bs.LastMinioScan = LastMinioScan.Format(DateTimeFormat) + bs.LastMinioScan = LastMinioScan } func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { @@ -244,7 +302,14 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { err error ) - rows, err = db.Table("reference_objects"). + alrdI := ctx.Value(AllocationListRequestDataKey) + if alrdI == nil { + Logger.Error("loadAllocationStats err: alrd should not be nil", zap.Any("err", err)) + return + } + alrd := alrdI.(RequestData) + + rows, err = db.Table("reference_objects").Offset(alrd.Offset).Limit(alrd.Limit). Select(` reference_objects.allocation_id, SUM(reference_objects.size) as files_size, @@ -292,6 +357,16 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { zap.Error(err)) return } + + var count int64 + err = db.Table("reference_objects").Where("deleted_at is null").Count(&count).Error + if err != nil { + Logger.Error("loadAllocationStats err where deleted_at is nul", zap.Any("err", err)) + return + } + + pagination := GeneratePagination(alrd.Page, alrd.Limit, alrd.Offset, int(count)) + bs.AllocationListPagination = *pagination } func (bs *BlobberStats) loadChallengeStats(ctx context.Context) { diff --git a/code/go/0chain.net/blobbercore/stats/challengestats.go b/code/go/0chain.net/blobbercore/stats/challengestats.go new file mode 100644 index 000000000..18c1f926b --- /dev/null +++ b/code/go/0chain.net/blobbercore/stats/challengestats.go @@ -0,0 +1,52 @@ +package stats + +import ( + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "gorm.io/datatypes" + "time" +) + +func (ChallengeEntity) TableName() string { + return "challenges" +} + +type ChallengeEntity struct { + ChallengeID string `json:"id" gorm:"column:challenge_id;primary_key"` + PrevChallengeID string `json:"prev_id" gorm:"column:prev_challenge_id"` + RandomNumber int64 `json:"seed" gorm:"column:seed"` + AllocationID string `json:"allocation_id" gorm:"column:allocation_id"` + AllocationRoot string `json:"allocation_root" gorm:"column:allocation_root"` + RespondedAllocationRoot string `json:"responded_allocation_root" gorm:"column:responded_allocation_root"` + Status int `json:"status" gorm:"column:status"` + Result int `json:"result" gorm:"column:result"` + StatusMessage string `json:"status_message" gorm:"column:status_message"` + CommitTxnID string `json:"commit_txn_id" gorm:"column:commit_txn_id"` + BlockNum int64 `json:"block_num" gorm:"column:block_num"` + ValidationTicketsString datatypes.JSON `json:"-" gorm:"column:validation_tickets"` + ValidatorsString datatypes.JSON `json:"-" gorm:"column:validators"` + LastCommitTxnList datatypes.JSON `json:"-" gorm:"column:last_commit_txn_ids"` + RefID int64 `json:"-" gorm:"column:ref_id"` + LastCommitTxnIDs []string `json:"last_commit_txn_ids" gorm:"-"` + ObjectPathString datatypes.JSON `json:"-" gorm:"column:object_path"` + ObjectPath *reference.ObjectPath `json:"object_path" gorm:"-"` + CreatedAt time.Time `gorm:"created_at"` + UpdatedAt time.Time `gorm:"updated_at"` +} + +func getAllFailedChallenges(offset, limit int) ([]ChallengeEntity, int, error) { + db := datastore.GetStore().GetDB() + crs := []ChallengeEntity{} + err := db.Offset(offset).Limit(limit).Order("challenge_id DESC").Table(ChallengeEntity{}.TableName()).Find(&crs, ChallengeEntity{Result: 2}).Error + if err != nil { + return nil, 0, err + } + + var count int64 + err = db.Table(ChallengeEntity{}.TableName()).Where("result = ?", 2).Count(&count).Error + if err != nil { + return nil, 0, err + } + + return crs, int(count), nil +} diff --git a/code/go/0chain.net/blobbercore/stats/dbstats.go b/code/go/0chain.net/blobbercore/stats/dbstats.go new file mode 100644 index 000000000..f348c23d1 --- /dev/null +++ b/code/go/0chain.net/blobbercore/stats/dbstats.go @@ -0,0 +1,11 @@ +package stats + +import ( + "database/sql" +) + +// DBStats contains database statistics. +type DBStats struct { + sql.DBStats + Status string +} diff --git a/code/go/0chain.net/blobbercore/stats/filestats.go b/code/go/0chain.net/blobbercore/stats/filestats.go index e2e49e74f..b6aebb826 100644 --- a/code/go/0chain.net/blobbercore/stats/filestats.go +++ b/code/go/0chain.net/blobbercore/stats/filestats.go @@ -2,9 +2,7 @@ package stats import ( "context" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "gorm.io/gorm" ) diff --git a/code/go/0chain.net/blobbercore/stats/handler.go b/code/go/0chain.net/blobbercore/stats/handler.go index 04614a126..3ca5e9912 100644 --- a/code/go/0chain.net/blobbercore/stats/handler.go +++ b/code/go/0chain.net/blobbercore/stats/handler.go @@ -3,15 +3,14 @@ package stats import ( "context" "fmt" - "html/template" - "net/http" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" - "go.uber.org/zap" + "html/template" + "net/http" + "time" ) func byteCountIEC(b int64) string { @@ -39,146 +38,224 @@ var funcMap = template.FuncMap{ "write_size": func(readCount int64) string { return byteCountIEC(readCount) }, + "byte_count_in_string": func(byteValue int64) string { + return byteCountIEC(byteValue) + }, + "time_in_string": func(timeValue time.Time) string { + if timeValue.IsZero() { + return "-" + } + return timeValue.Format(DateTimeFormat) + }, } -const tpl = ` - - - Blobber Diagnostics - - -

- Blobber Stats -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ID{{ .ClientID }}
PublicKey{{ .PublicKey }}
Allocations{{ .NumAllocation }}
Allocated size (bytes){{ .AllocatedSize }}
Used Size (bytes){{ .UsedSize }}
Actual Disk Usage (bytes){{ .DiskSizeUsed }}
Cloud Files Size (bytes){{ .CloudFilesSize }}
Cloud Files Count{{ .CloudTotalFiles }}
Last Minio Scan{{ .LastMinioScan }}
Num of files{{ .NumWrites }}
Blocks Written{{ .BlockWrites }}
Blocks Read{{ .NumReads }}
Total Challenges{{ .TotalChallenges }}
Open Challenges{{ .OpenChallenges }}
Passed Challenges{{ .SuccessChallenges }}
Failed Challenges{{ .FailedChallenges }}
Redeemed Challenges{{ .RedeemedChallenges }}
Redeemed Challenges{{ .RedeemedChallenges }}
- - - - - - - - - -
Configurations
Capacity{{ .Capacity }}
Read price{{ .ReadPrice }}
Write price{{ .WritePrice }}
Min lock demand{{ .MinLockDemand }}
Max offer duration{{ .MaxOfferDuration }}
Challenge completion_time{{ .ChallengeCompletionTime }}
Read lock timeout{{ .ReadLockTimeout }}
Write lock timeout{{ .WriteLockTimeout }}
- - - - - - - - - - - - - - -
Read markers
Pending{{ .ReadMarkers.Pending }} (64 KB blocks){{ read_size .ReadMarkers.Pending }}
Redeemed{{ .ReadMarkers.Redeemed }} (64 KB blocks){{ read_size .ReadMarkers.Redeemed }}
- - - - - - - - - - - - - - - - - - - -
Write markers
Accepted{{ .WriteMarkers.Accepted.Count }} markers{{ write_size .WriteMarkers.Accepted.Size }}
Committed{{ .WriteMarkers.Committed.Count }} markers{{ write_size .WriteMarkers.Committed.Size }}
Failed{{ .WriteMarkers.Failed.Count }} markers{{ write_size .WriteMarkers.Failed.Size }}
- - +const tpl = ` + + + + + + + + + + + + + + + +
SummaryConfigurationsMarkersInfra StatsDatabase
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Allocations{{ .NumAllocation }}
Allocated size{{ byte_count_in_string .AllocatedSize }}
Used Size{{ byte_count_in_string .UsedSize }}
Actual Disk Usage{{ byte_count_in_string .DiskSizeUsed }}
Files Size{{ byte_count_in_string .FilesSize }}
Cloud Files Size{{ byte_count_in_string .CloudFilesSize }}
Cloud Files Count{{ .CloudTotalFiles }}
Last Minio Scan{{ time_in_string .LastMinioScan }}
Num of files{{ .NumWrites }}
Blocks Written{{ .BlockWrites }}
Blocks Read{{ .NumReads }}
Total Challenges{{ .TotalChallenges }}
Open Challenges{{ .OpenChallenges }}
Passed Challenges{{ .SuccessChallenges }}
Failed Challenges{{ .FailedChallenges }}
Redeemed Challenges{{ .RedeemedChallenges }}
Redeemed Challenges{{ .RedeemedChallenges }}
+ 		+ + + + + + + + + +
Capacity{{ byte_count_in_string .Capacity }}
Read price{{ .ReadPrice }}
Write price{{ .WritePrice }}
Min lock demand{{ .MinLockDemand }}
Max offer duration{{ .MaxOfferDuration }}
Challenge completion_time{{ .ChallengeCompletionTime }}
Read lock timeout{{ .ReadLockTimeout }}
Write lock timeout{{ .WriteLockTimeout }}
+ 		+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Read Markers
Pending{{ .ReadMarkers.Pending }} (64 KB blocks){{ read_size .ReadMarkers.Pending }}
Redeemed{{ .ReadMarkers.Redeemed }} (64 KB blocks){{ read_size .ReadMarkers.Redeemed }}
Write Markers
Accepted{{ .WriteMarkers.Accepted.Count }} markers{{ write_size .WriteMarkers.Accepted.Size }}
Committed{{ .WriteMarkers.Committed.Count }} markers{{ write_size .WriteMarkers.Committed.Size }}
Failed{{ .WriteMarkers.Failed.Count }} markers{{ write_size .WriteMarkers.Failed.Size }}
+ 		+ + + + + + + + + + + + + + + + + + + + + +
CPU{{ .InfraStats.CPUs }}
Go Routines{{ .InfraStats.NumberOfGoroutines }}
Heap Sys{{ byte_count_in_string .InfraStats.HeapSys }}
Heap Alloc{{ byte_count_in_string .InfraStats.HeapAlloc }}
Is Active On Chain{{ .InfraStats.ActiveOnChain }}
+ 		+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Open Connections{{ .DBStats.OpenConnections }}
Connections In Use{{ .DBStats.InUse }}
Idle Connections{{ .DBStats.Idle }}
Connection Wait Count{{ .DBStats.WaitCount }}
Connection Wait Duration{{ .DBStats.WaitDuration }}
Max Open Connections{{ .DBStats.MaxOpenConnections }}
Connection Max Idle Closed{{ .DBStats.MaxIdleClosed }}
Connection Max Idle Time Closed{{ .DBStats.MaxIdleTimeClosed }}
Connection Max Lifetime Closed{{ .DBStats.MaxLifetimeClosed }}
Database Status{{ .DBStats.Status }}
+
+ +

+ Allocation Stats +

-

- Allocation Stats -

-

Note: You might not see stats for all allocations. Allocations that have no data will not be collecting stats

- - +

Note: You might not see stats for all allocations. Allocations that have no data will not be collecting stats

+ +
+
+ @@ -192,14 +269,14 @@ const tpl = ` - - {{range .AllocationStats}} + + {{range .AllocationStats}} - + - - - + + + @@ -209,70 +286,203 @@ const tpl = ` - - + + - - {{end}} -
ID Used Size (bytes) Actual Disk Usage (bytes) Failed Challenges Redeemed Challenges Expiration
{{ .AllocationID }}{{ .UsedSize }}{{ .DiskSizeUsed }}{{ .TempFolderSize }}{{ byte_count_in_string .UsedSize }}{{ byte_count_in_string .DiskSizeUsed }}{{ byte_count_in_string .TempFolderSize }} {{ .NumWrites }} {{ .BlockWrites }} {{ .NumReads }} {{ .FailedChallenges }} {{ .RedeemedChallenges }} {{ .Expiration }}
- - {{ if .ReadMarkers }} - - - - - - - - - - - - {{ else }} - - {{ end }} -
Read markers
Pending{{ .ReadMarkers.Pending }} (64 KB blocks){{ read_size .ReadMarkers.Pending }}
Redeemed{{ .ReadMarkers.Redeemed }} (64 KB blocks){{ read_size .ReadMarkers.Redeemed }}
No read markers yet.
+ + {{ if .ReadMarkers }} + + + + + + + + + + + + {{ else }} + + {{ end }} +
Read markers
Pending{{ .ReadMarkers.Pending }} (64 KB blocks){{ read_size .ReadMarkers.Pending }}
Redeemed{{ .ReadMarkers.Redeemed }} (64 KB blocks){{ read_size .ReadMarkers.Redeemed }}
No read markers yet.
 - - {{ if .WriteMarkers }} - - - - - - - - - - - - - - - - - {{ else }} - - {{ end }} -
Write markers
Accepted{{ .WriteMarkers.Accepted.Count }} markers{{ write_size .WriteMarkers.Accepted.Size }}
Committed{{ .WriteMarkers.Committed.Count }} markers{{ write_size .WriteMarkers.Committed.Size }}
Failed{{ .WriteMarkers.Failed.Count }} markers{{ write_size .WriteMarkers.Failed.Size }}
No write markers yet
+ + {{ if .WriteMarkers }} + + + + + + + + + + + + + + + + + {{ else }} + + {{ end }} +
Write markers
Accepted{{ .WriteMarkers.Accepted.Count }} markers{{ write_size .WriteMarkers.Accepted.Size }}
Committed{{ .WriteMarkers.Committed.Count }} markers{{ write_size .WriteMarkers.Committed.Size }}
Failed{{ .WriteMarkers.Failed.Count }} markers{{ write_size .WriteMarkers.Failed.Size }}
No write markers yet
- - + + {{end}} + + +
+ +
+ +
+ +

+ Failed Challenges +

+ + + + + + + + + + {{range .FailedChallengeList}} + + + + + + + + {{end}} + +
IDOther IDsInfoFailed AtMessage
{{ .ChallengeID }} + + + + + + + + + + + + + +
Allocation ID{{ .AllocationID }}
Allocation Root{{ .AllocationRoot }}
Previous Challenge ID{{ .PrevChallengeID }}
+ 		+ + + + + + + + + + + + + +
Block Number{{ .BlockNum }}
Result{{ .Result }}
Status{{ .Status }}
+ 		{{ time_in_string .UpdatedAt }}{{ .StatusMessage }}
+ +
+ +
+ ` func StatsHandler(w http.ResponseWriter, r *http.Request) { t := template.Must(template.New("diagnostics").Funcs(funcMap).Parse(tpl)) ctx := datastore.GetStore().CreateTransaction(r.Context()) + ctx = setStatsRequestDataInContext(r, ctx) db := datastore.GetStore().GetTransaction(ctx) defer db.Rollback() bs := LoadBlobberStats(ctx) + err := t.Execute(w, bs) if err != nil { Logger.Error("Error in executing the template", zap.Error(err)) } } +func setStatsRequestDataInContext(r *http.Request, ctx context.Context) context.Context { + ctx = context.WithValue(ctx, HealthDataKey, r.Header.Get(HealthDataKey.String())) + + allocationPage := r.URL.Query().Get("alp") + alPageLimitOffset, err := GetPageLimitOffsetFromRequestData(allocationPage) + if err != nil { + Logger.Error("setStatsRequestDataInContext", zap.Error(err)) + return ctx + } + alrd := RequestData{ + Page: alPageLimitOffset.Page, + Limit: alPageLimitOffset.Limit, + Offset: alPageLimitOffset.Offset, + } + ctx = context.WithValue(ctx, AllocationListRequestDataKey, alrd) + + failedChallengePage := r.URL.Query().Get("fcp") + fcPageLimitOffset, err := GetPageLimitOffsetFromRequestData(failedChallengePage) + if err != nil { + Logger.Error("setStatsRequestDataInContext", zap.Error(err)) + return ctx + } + fcrd := RequestData{ + Page: fcPageLimitOffset.Page, + Limit: fcPageLimitOffset.Limit, + Offset: fcPageLimitOffset.Offset, + } + ctx = context.WithValue(ctx, FailedChallengeRequestDataKey, fcrd) + + return ctx +} + func StatsJSONHandler(ctx context.Context, r *http.Request) (interface{}, error) { ctx = datastore.GetStore().CreateTransaction(ctx) db := datastore.GetStore().GetTransaction(ctx) diff --git a/code/go/0chain.net/blobbercore/stats/infrastats.go b/code/go/0chain.net/blobbercore/stats/infrastats.go new file mode 100644 index 000000000..271f4e827 --- /dev/null +++ b/code/go/0chain.net/blobbercore/stats/infrastats.go @@ -0,0 +1,21 @@ +package stats + +type contextKey string + +func (c contextKey) String() string { + return string(c) +} + +const ( + HealthDataKey = contextKey("health") + FailedChallengeRequestDataKey = contextKey("fcrd") + AllocationListRequestDataKey = contextKey("alrd") +) + +type InfraStats struct { + CPUs int + NumberOfGoroutines int + HeapAlloc int64 + HeapSys int64 + ActiveOnChain string +} diff --git a/code/go/0chain.net/blobbercore/stats/pagination.go b/code/go/0chain.net/blobbercore/stats/pagination.go new file mode 100644 index 000000000..69c9f4cf7 --- /dev/null +++ b/code/go/0chain.net/blobbercore/stats/pagination.go @@ -0,0 +1,108 @@ +package stats + +import ( + "strconv" +) + +type RequestData struct { + Page int + Limit int + Offset int +} + +type Pagination struct { + TotalItems int + CurrentPage int + Limit int + TotalPages int + Offset int + PrevPage int + NextPage int + HasPrev bool + HasNext bool + HasDivider bool + CurrentPageIsInDivider bool + FirstFew []int + LastFew []int +} + +type PageLimitOffsetData struct { + Page int + Limit int + Offset int +} + +func GetPageLimitOffsetFromRequestData(page string) (*PageLimitOffsetData, error) { + if page == "" { + page = "1" + } + pageInt, err := strconv.Atoi(page) + if err != nil { + return nil, err + } + if pageInt < 1 { + pageInt = 1 + } + + limitInt := 20 + + offset := 0 + if pageInt > 1 { + offset = (pageInt - 1) * limitInt + } + + out := PageLimitOffsetData{ + Page: pageInt, + Limit: limitInt, + Offset: offset, + } + + return &out, nil +} + +func GeneratePagination(page, limit, offset, totalItems int) *Pagination { + p := Pagination{TotalItems: totalItems, CurrentPage: page, Limit: limit, Offset: offset} + if totalItems < 1 { + return &p + } + + if offset >= totalItems { + return &p + } + + p.TotalPages = totalItems / limit + if totalItems%limit > 0 { + p.TotalPages = p.TotalPages + 1 + } + + if p.TotalPages > p.CurrentPage { + p.HasNext = true + p.NextPage = p.CurrentPage + 1 + } + + if p.CurrentPage > 1 { + p.HasPrev = true + p.PrevPage = p.CurrentPage - 1 + } + + if p.TotalPages > 5 { + p.HasDivider = true + for i := 1; i < 4; i++ { + p.FirstFew = append(p.FirstFew, i) + } + count := 0 + for i := p.TotalPages; i > 5; i-- { + if count > 3 { + break + } + p.LastFew = append(p.LastFew, i) + count++ + } + + if p.CurrentPage > 5 && p.CurrentPage <= (p.TotalPages-3) { + p.CurrentPageIsInDivider = true + } + } + + return &p +} diff --git a/code/go/0chain.net/core/common/time.go b/code/go/0chain.net/core/common/time.go index 47da3f17c..d35503638 100644 --- a/code/go/0chain.net/core/common/time.go +++ b/code/go/0chain.net/core/common/time.go @@ -4,6 +4,9 @@ import ( "time" ) +//DateTimeFormat - the format in which the date time fields should be displayed in the UI +var DateTimeFormat = "2006-01-02T15:04:05+00:00" + /*Timestamp - just a wrapper to control the json encoding */ type Timestamp int64 @@ -22,4 +25,3 @@ func Within(ts int64, seconds int64) bool { func ToTime(ts Timestamp) time.Time { return time.Unix(int64(ts), 0) } - From e9ad91e7d028088695112dc75fbadc42f942fdfa Mon Sep 17 00:00:00 2001 From: prince Date: Wed, 1 Dec 2021 11:45:56 +0530 Subject: [PATCH 066/161] in case of listing for newallocation (#447) if we try to get file ref from db for path / it is not available. in that case we ignore the error which result in nil fileref and nil pointer error. to resolve that condition in case of nil fileref and path / we can use / directing to get the data and avoid nil pointer issue. Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar --- .../0chain.net/blobbercore/handler/storage_handler.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index ace2eb2ce..e821599b4 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -455,7 +455,15 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } } - dirref, err := reference.GetRefWithChildren(ctx, allocationID, fileref.Path) + // when '/' is not available in database we ignore 'record not found' error. which results into nil fileRef + // to handle that condition use filePath '/' while file ref is nil and path is '/' + filePath := path + if fileref != nil { + filePath = fileref.Path + } else if path != "/" { + return nil, common.NewError("invalid_parameters", "Invalid path: ref not found ") + } + dirref, err := reference.GetRefWithChildren(ctx, allocationID, filePath) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid path. "+err.Error()) } From 500fd52abc839539ddb6f6b0d27a758f77f49fd1 Mon Sep 17 00:00:00 2001 From: prince Date: Sat, 4 Dec 2021 17:37:49 +0530 Subject: [PATCH 067/161] Create dir logic added if given path is not available. (#446) * Create dir logic added if given path is not available. Signed-off-by: Pradip Parmar * copy_ok testcase query mock change. Signed-off-by: Pradip Parmar * removed createdir methor from obejct operation and added mkdir in ref. Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * testcase fix Signed-off-by: Pradip Parmar * allocation id issue resolved Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Lz --- .../blobbercore/handler/handler_test.go | 7 +++ .../handler/object_operation_handler.go | 2 +- .../0chain.net/blobbercore/reference/ref.go | 45 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 17b35901a..1375092cd 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -907,6 +907,13 @@ func TestHandlers_Requiring_Signature(t *testing.T) { AddRow(reference.DIRECTORY), ) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(aa, aa). + WillReturnRows( + sqlmock.NewRows([]string{"type"}). + AddRow(reference.DIRECTORY), + ) + mock.ExpectExec(`INSERT INTO "allocation_connections"`). WithArgs(aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 568ce8fb3..4c008ae94 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -934,7 +934,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") } - destRef, err = reference.GetReference(ctx, allocationID, destPath) + destRef, err = reference.Mkdir(ctx, allocationID, destPath) if err != nil || destRef.Type != reference.DIRECTORY { return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") } diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index adcaebf48..03e238ad0 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -3,6 +3,7 @@ package reference import ( "context" "encoding/json" + "errors" "math" "path/filepath" "reflect" @@ -171,6 +172,50 @@ func (r *Ref) SetAttributes(attr *Attributes) (err error) { return } +// Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be returned +func Mkdir(ctx context.Context, allocationID string, destpath string) (*Ref, error) { + var dirRef *Ref + db := datastore.GetStore().GetTransaction(ctx) + // cleaning path to avoid edge case issues: append '/' prefix if not added and removing suffix '/' if added + destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") + dirs := strings.Split(destpath, "/") + + for i := range dirs { + currentPath := filepath.Join("/", filepath.Join(dirs[:i+1]...)) + ref, err := GetReference(ctx, allocationID, currentPath) + if err == nil { + ref.AddChild(dirRef) + dirRef = ref + continue + } + + if !errors.Is(err, gorm.ErrRecordNotFound) { + // unexpected sql error + return nil, err + } + + // dir doesn't exists , create it + newRef := NewDirectoryRef() + newRef.AllocationID = allocationID + newRef.Path = currentPath + newRef.ParentPath = filepath.Join("/", filepath.Join(dirs[:i]...)) + newRef.Name = dirs[i] + newRef.Type = DIRECTORY + newRef.PathLevel = i + 1 + newRef.LookupHash = GetReferenceLookup(allocationID, newRef.Path) + err = db.Create(newRef).Error + if err != nil { + return nil, err + } + + newRef.AddChild(dirRef) + dirRef = newRef + } + + return dirRef, nil + +} + // GetReference get FileRef with allcationID and path from postgres func GetReference(ctx context.Context, allocationID string, path string) (*Ref, error) { ref := &Ref{} From 2cd899d048456e78fd82f0c54c9fc2b2d1f0faea Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 6 Dec 2021 22:09:11 +0000 Subject: [PATCH 068/161] Feature/use system tests action (#456) * replace existing system tests trigger and poll with native action * read correct var * add substitution correctly * allow tests to be skipped * re-order params to push blobber/validator options first * test commit * fic formatting * retry failures --- .github/workflows/system_tests.yml | 88 ++++++++++++++++-------------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index ae07fd1af..299c7157b 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -11,6 +11,14 @@ on: description: 'system_tests branch. Containing the tests you wish to run' default: 'master' required: true + blobber_image: + description: 'blobber DOCKER IMAGE to deploy' + default: 'staging' + required: true + validator_image: + description: 'validator DOCKER IMAGE to deploy' + default: 'staging' + required: true zbox_cli_branch: description: '0Box CLI (branch or commit SHA) which the tests will use' default: 'staging' @@ -27,14 +35,6 @@ on: description: 'sharder DOCKER IMAGE to deploy' default: 'staging' required: true - blobber_image: - description: 'blobber DOCKER IMAGE to deploy' - default: 'staging' - required: true - validator_image: - description: 'validator DOCKER IMAGE to deploy' - default: 'staging' - required: true zbox_image: description: '0box DOCKER IMAGE to deploy' default: 'staging' @@ -53,20 +53,18 @@ jobs: timeout-minutes: 40 steps: - - uses: jwalton/gh-find-current-pr@v1 + - name: "Get current PR" + uses: jwalton/gh-find-current-pr@v1 id: findPr with: github-token: ${{ github.token }} - - name: Extract branch name - shell: bash + - name: "Setup Test Run" run: | - echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") - echo "##[set-output name=pr_number;]$(echo $pull_number)" - id: extract_branch + echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV + echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV - - name: Set PR Status Pending + - name: "Set PR Status Pending" uses: niteoweb/pull_request_status_action@v1.0.0 if: steps.findPr.outputs.number with: @@ -79,34 +77,40 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - - uses: stewartie4/trigger-workflow-and-wait@master - name: "Run System Tests" + - name: "Deploy 0Chain" + if: ${{ github.event.inputs.skip_tests == 'FALSE' }} + uses: 0chain/actions/deploy-0chain@master + with: + kube_config: ${{ secrets[format('DEV{0}KC', env.RUNNER_NUMBER)] }} + teardown_condition: "TESTS_PASSED" + miner_image: ${{ github.event.inputs.miner_image }} + sharder_image: ${{ github.event.inputs.sharder_image }} + blobber_image: ${{ github.event.inputs.blobber_image }} + validator_image: ${{ github.event.inputs.validator_image }} + zbox_image: ${{ github.event.inputs.zbox_image }} + zblock_image: ${{ github.event.inputs.zblock_image }} + zdns_image: staging + explorer_image: latest + zproxy_image: staging + zsearch_image: staging + blobber_stake_image: latest + + - name: "Run System tests" if: ${{ github.event.inputs.skip_tests == 'FALSE' }} + uses: 0chain/actions/run-system-tests@master with: - owner: 0chain - repo: system_test - github_token: ${{ secrets.SVC_ACCOUNT_SECRET }} - workflow_file_name: ci.yml - ref: ${{ github.event.inputs.system_tests_branch }} - wait_interval: 60 - inputs: > - { - "test_run_description" : "Triggered by [${{ github.event.repository.name }}] repo, branch [${{ steps.extract_branch.outputs.branch }}], URL [https://github.com/0chain/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}]", - "zbox_cli_branch" : "${{github.event.inputs.zbox_cli_branch}}", - "zwallet_cli_branch" : "${{github.event.inputs.zwallet_cli_branch}}", - "miner_image": "${{github.event.inputs.miner_image}}", - "sharder_image": "${{github.event.inputs.sharder_image}}", - "blobber_image": "${{github.event.inputs.blobber_image}}", - "validator_image": "${{github.event.inputs.validator_image}}", - "zbox_image": "${{github.event.inputs.zbox_image}}", - "zblock_image": "${{github.event.inputs.zblock_image}}" - } - propagate_failure: true - trigger_workflow: true - wait_workflow: true - github_user: 'service-0chain' + system_tests_branch: ${{ github.event.inputs.system_tests_branch }} + network: ${{ env.NETWORK_URL }} + zbox_cli_branch: ${{ github.event.inputs.zbox_cli_branch }} + zwallet_cli_branch: ${{ github.event.inputs.zwallet_cli_branch }} + smart_contract_owner_wallet_json: ${{ secrets.SMART_CONTRACT_OWNER_WALLET_JSON }} + svc_account_secret: ${{ github.token }} + deploy_report_page: false + archive_results: true + run_flaky_tests: false + retry_failures: true - - name: Set PR Status Success + - name: "Set PR Status Success" if: ${{ success() && steps.findPr.outputs.number }} uses: niteoweb/pull_request_status_action@v1.0.0 with: @@ -119,7 +123,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - - name: Set PR Status Failed + - name: "Set PR Status Failed" if: ${{ failure() && steps.findPr.outputs.number }} uses: niteoweb/pull_request_status_action@v1.0.0 with: From 15777e347b6cec666940c4efc312e97462e4ef1c Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 8 Dec 2021 07:41:04 +0800 Subject: [PATCH 069/161] feat(wasmsdk):0chain/gosdk#294 fixed cors on blobber hanlders (#457) * feat(wasmsdk):0chain/gosdk#294 fixed cors on blobber hanlders * feat(wasmsdk):0chain/gosdk#294 fixed cors on blobber hanlders * feat(wasmsdk):0chain/gosdk#294 fixed cors on handlers * feat(wasmsdk):0chain/gosdk#294 fixed cors on handlers --- .../0chain.net/blobbercore/handler/handler.go | 2 +- .../handler/handler_middlewares.go | 44 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 5ca61d8bf..fe14753ac 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -41,7 +41,7 @@ func GetMetaDataStore() datastore.Store { /*SetupHandlers sets up the necessary API end points */ func SetupHandlers(r *mux.Router) { - r.Use(useRecovery, useCORS(), common.UseUserRateLimit) + r.Use(useRecovery, useCors, common.UseUserRateLimit) //object operations r.HandleFunc("/v1/file/upload/{allocation}", common.ToJSONResponse(WithConnection(UploadHandler))) diff --git a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go index 8ea009a4f..7b462589d 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go +++ b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go @@ -2,27 +2,45 @@ package handler import ( "net/http" + "strings" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/gorilla/handlers" "go.uber.org/zap" ) -func useCORS() func(http.Handler) http.Handler { - headersOk := handlers.AllowedHeaders([]string{ - "X-Requested-With", "X-App-Client-ID", - "X-App-Client-Key", "Content-Type", - "X-App-Client-Signature", - }) +func useCors(h http.Handler) http.Handler { + allowedHeaders := []string{ + "X-Requested-With", "Content-Type", + "X-App-Client-ID", "X-App-Client-Key", "X-App-Client-Signature", + "Access-Control-Allow-origin", "Access-Control-Request-Method", + } + + allowedOrigins := []string{"*"} + + allowedMethods := []string{"GET", "HEAD", "POST", "PUT", + "DELETE", "OPTIONS"} + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + logging.Logger.Error("[recover]http", zap.String("url", r.URL.String()), zap.Any("err", err)) + } + }() - // Allow anybody to access API. - // originsOk := handlers.AllowedOriginValidator(isValidOrigin) - originsOk := handlers.AllowedOrigins([]string{"*"}) + w.Header().Add("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", ")) + w.Header().Add("Access-Control-Allow-Origin", strings.Join(allowedOrigins, ", ")) + w.Header().Add("Access-Control-Allow-Methods", strings.Join(allowedMethods, ", ")) + w.Header().Add("Access-Control-Allow-Credentials", "true") - methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", - "DELETE", "OPTIONS"}) + // return directly for preflight request + if r.Method == http.MethodOptions { + w.Header().Add("Access-Control-Max-Age", "3600") + w.WriteHeader(http.StatusNoContent) + return + } - return handlers.CORS(originsOk, headersOk, methodsOk) + h.ServeHTTP(w, r) + }) } func useRecovery(h http.Handler) http.Handler { From 27b1edaeaf1d7409e2ebb52a1484e56808730778 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 8 Dec 2021 21:00:43 +0800 Subject: [PATCH 070/161] fix cors and max_file_size checking (#458) * feat(wasmsdk):0chain/gosdk#313 fixed cors on handlers * feat(upload):ignore file size checking if max_file_size is 0 * feat(upload):rejected upload if actual size is greater than max_file_size * fix(upload): renamed changeProcesser with fileChanger --- .../handler/file_command_insert.go | 50 +++++++++++-------- .../handler/file_command_update.go | 11 +++- .../0chain.net/blobbercore/handler/handler.go | 12 ++--- config/0chain_blobber.yaml | 3 +- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_insert.go b/code/go/0chain.net/blobbercore/handler/file_command_insert.go index 81e5de259..1a815d3d7 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_insert.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_insert.go @@ -18,7 +18,7 @@ import ( // InsertFileCommand command for inserting file type InsertFileCommand struct { allocationChange *allocation.AllocationChange - changeProcessor *allocation.UpdateFileChanger + fileChanger *allocation.UpdateFileChanger } // IsAuthorized validate request. @@ -28,25 +28,25 @@ func (cmd *InsertFileCommand) IsAuthorized(ctx context.Context, req *http.Reques return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") } - changeProcessor := &allocation.UpdateFileChanger{} + fileChanger := &allocation.UpdateFileChanger{} uploadMetaString := req.FormValue("uploadMeta") - err := json.Unmarshal([]byte(uploadMetaString), changeProcessor) + err := json.Unmarshal([]byte(uploadMetaString), fileChanger) if err != nil { return common.NewError("invalid_parameters", "Invalid parameters. Error parsing the meta data for upload."+err.Error()) } - exisitingFileRef, _ := reference.GetReference(ctx, allocationObj.ID, changeProcessor.Path) + exisitingFileRef, _ := reference.GetReference(ctx, allocationObj.ID, fileChanger.Path) if exisitingFileRef != nil { return common.NewError("duplicate_file", "File at path already exists") } - if changeProcessor.ChunkSize <= 0 { - changeProcessor.ChunkSize = fileref.CHUNK_SIZE + if fileChanger.ChunkSize <= 0 { + fileChanger.ChunkSize = fileref.CHUNK_SIZE } - cmd.changeProcessor = changeProcessor + cmd.fileChanger = fileChanger return nil } @@ -55,6 +55,7 @@ func (cmd *InsertFileCommand) IsAuthorized(ctx context.Context, req *http.Reques func (cmd *InsertFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { result := blobberhttp.UploadResult{} + result.Filename = cmd.fileChanger.Filename origfile, _, err := req.FormFile("uploadFile") if err != nil { @@ -62,31 +63,36 @@ func (cmd *InsertFileCommand) ProcessContent(ctx context.Context, req *http.Requ } defer origfile.Close() - fileInputData := &filestore.FileInputData{Name: cmd.changeProcessor.Filename, Path: cmd.changeProcessor.Path, OnCloud: false} + // rejected it on first chunk if actualsize is greater than max_file_size + if config.Configuration.MaxFileSize > 0 && cmd.fileChanger.ActualSize > config.Configuration.MaxFileSize { + return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") + } + + fileInputData := &filestore.FileInputData{Name: cmd.fileChanger.Filename, Path: cmd.fileChanger.Path, OnCloud: false} fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) if err != nil { return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) } - result.Filename = cmd.changeProcessor.Filename result.Hash = fileOutputData.ContentHash result.MerkleRoot = fileOutputData.MerkleRoot result.Size = fileOutputData.Size - if len(cmd.changeProcessor.Hash) > 0 && cmd.changeProcessor.Hash != fileOutputData.ContentHash { + if len(cmd.fileChanger.Hash) > 0 && cmd.fileChanger.Hash != fileOutputData.ContentHash { return result, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") } - if len(cmd.changeProcessor.MerkleRoot) > 0 && cmd.changeProcessor.MerkleRoot != fileOutputData.MerkleRoot { + if len(cmd.fileChanger.MerkleRoot) > 0 && cmd.fileChanger.MerkleRoot != fileOutputData.MerkleRoot { return result, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") } - if fileOutputData.Size > config.Configuration.MaxFileSize { + //max_file_size = 0 means file size is unlimited + if fileOutputData.Size > config.Configuration.MaxFileSize && config.Configuration.MaxFileSize > 0 { return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") } - cmd.changeProcessor.Hash = fileOutputData.ContentHash - cmd.changeProcessor.MerkleRoot = fileOutputData.MerkleRoot - cmd.changeProcessor.AllocationID = allocationObj.ID - cmd.changeProcessor.Size = fileOutputData.Size + cmd.fileChanger.Hash = fileOutputData.ContentHash + cmd.fileChanger.MerkleRoot = fileOutputData.MerkleRoot + cmd.fileChanger.AllocationID = allocationObj.ID + cmd.fileChanger.Size = fileOutputData.Size allocationSize := fileOutputData.Size @@ -114,18 +120,18 @@ func (cmd *InsertFileCommand) ProcessThumbnail(ctx context.Context, req *http.Re defer thumbfile.Close() - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.changeProcessor.Path} + thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, thumbInputData, thumbfile, connectionObj.ConnectionID) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) } - if len(cmd.changeProcessor.ThumbnailHash) > 0 && cmd.changeProcessor.ThumbnailHash != thumbOutputData.ContentHash { + if len(cmd.fileChanger.ThumbnailHash) > 0 && cmd.fileChanger.ThumbnailHash != thumbOutputData.ContentHash { return common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") } - cmd.changeProcessor.ThumbnailHash = thumbOutputData.ContentHash - cmd.changeProcessor.ThumbnailSize = thumbOutputData.Size - cmd.changeProcessor.ThumbnailFilename = thumbInputData.Name + cmd.fileChanger.ThumbnailHash = thumbOutputData.ContentHash + cmd.fileChanger.ThumbnailSize = thumbOutputData.Size + cmd.fileChanger.ThumbnailFilename = thumbInputData.Name } return nil @@ -134,6 +140,6 @@ func (cmd *InsertFileCommand) ProcessThumbnail(ctx context.Context, req *http.Re // UpdateChange add NewFileChange in db func (cmd *InsertFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { - connectionObj.AddChange(cmd.allocationChange, cmd.changeProcessor) + connectionObj.AddChange(cmd.allocationChange, cmd.fileChanger) return connectionObj.Save(ctx) } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index fbae2b6f6..b92672c5d 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -64,6 +64,13 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ result := blobberhttp.UploadResult{} + result.Filename = cmd.fileChanger.Filename + + // rejected it on first chunk if actualsize is greater than max_file_size + if config.Configuration.MaxFileSize > 0 && cmd.fileChanger.ActualSize > config.Configuration.MaxFileSize { + return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") + } + origfile, _, err := req.FormFile("uploadFile") if err != nil { return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) @@ -86,7 +93,6 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) } - result.Filename = cmd.fileChanger.Filename result.Hash = fileOutputData.ContentHash //result.MerkleRoot = fileOutputData.MerkleRoot result.Size = fileOutputData.Size @@ -110,7 +116,8 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") } - if fileOutputData.Size > config.Configuration.MaxFileSize { + //max_file_size = 0 means file size is unlimited + if fileOutputData.Size > config.Configuration.MaxFileSize && config.Configuration.MaxFileSize > 0 { return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index fe14753ac..17ab203f7 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -45,12 +45,12 @@ func SetupHandlers(r *mux.Router) { //object operations r.HandleFunc("/v1/file/upload/{allocation}", common.ToJSONResponse(WithConnection(UploadHandler))) - r.HandleFunc("/v1/file/download/{allocation}", common.ToByteStream(WithConnection(DownloadHandler))).Methods("POST") - r.HandleFunc("/v1/file/rename/{allocation}", common.ToJSONResponse(WithConnection(RenameHandler))).Methods("POST") + r.HandleFunc("/v1/file/download/{allocation}", common.ToByteStream(WithConnection(DownloadHandler))).Methods(http.MethodPost, http.MethodOptions) + r.HandleFunc("/v1/file/rename/{allocation}", common.ToJSONResponse(WithConnection(RenameHandler))).Methods(http.MethodPost, http.MethodOptions) r.HandleFunc("/v1/file/copy/{allocation}", common.ToJSONResponse(WithConnection(CopyHandler))) r.HandleFunc("/v1/file/attributes/{allocation}", common.ToJSONResponse(WithConnection(UpdateAttributesHandler))) - r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("POST") - r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods("DELETE") + r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods(http.MethodPost, http.MethodOptions) + r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods(http.MethodDelete, http.MethodOptions) r.HandleFunc("/v1/connection/commit/{allocation}", common.ToJSONResponse(WithConnection(CommitHandler))) r.HandleFunc("/v1/file/commitmetatxn/{allocation}", common.ToJSONResponse(WithConnection(CommitMetaTxnHandler))) @@ -61,11 +61,11 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/allocation", common.ToJSONResponse(WithConnection(AllocationHandler))) r.HandleFunc("/v1/file/meta/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(FileMetaHandler))) r.HandleFunc("/v1/file/stats/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(FileStatsHandler))) - r.HandleFunc("/v1/file/list/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ListHandler))).Methods("GET") + r.HandleFunc("/v1/file/list/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ListHandler))).Methods(http.MethodGet, http.MethodOptions) r.HandleFunc("/v1/file/objectpath/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler))) r.HandleFunc("/v1/file/referencepath/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler))) r.HandleFunc("/v1/file/objecttree/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler))) - r.HandleFunc("/v1/file/refs/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(RefsHandler))).Methods("GET") + r.HandleFunc("/v1/file/refs/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(RefsHandler))).Methods(http.MethodGet, http.MethodOptions) //admin related r.HandleFunc("/_debug", common.ToJSONResponse(DumpGoRoutines)) r.HandleFunc("/_config", common.ToJSONResponse(GetConfig)) diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index 2eb5bce8e..d741b8d28 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -36,7 +36,8 @@ challenge_completion_time: 2m # duration to complete a challenge # a task and redeem tokens, it should be big enough read_lock_timeout: 1m write_lock_timeout: 1m -max_file_size: 104857600 #10MB + +max_file_size: 104857600 #10MB,please set it with 0 if file size is unlimited # update_allocations_interval used to refresh known allocation objects from SC update_allocations_interval: 1m From d033a32ca45dba7a5d6aa7816f92c0d3105d7b69 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Fri, 10 Dec 2021 04:15:01 +0545 Subject: [PATCH 071/161] Remove max_file_size restriction (#460) * Remove max_file_size restriction * Fix undefined variable issue --- code/go/0chain.net/blobber/config.go | 1 - .../blobbercore/allocation/renamefilechange_test.go | 2 -- code/go/0chain.net/blobbercore/config/config.go | 1 - .../blobbercore/handler/file_command_add.go | 5 ----- .../blobbercore/handler/file_command_insert.go | 10 ---------- .../blobbercore/handler/file_command_update.go | 11 ----------- .../go/0chain.net/blobbercore/handler/handler_test.go | 2 -- config/0chain_blobber.yaml | 2 -- 8 files changed, 34 deletions(-) diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go index 2271040f5..0b4665494 100644 --- a/code/go/0chain.net/blobber/config.go +++ b/code/go/0chain.net/blobber/config.go @@ -51,7 +51,6 @@ func setupConfig() { config.Configuration.MinioUseSSL = viper.GetBool("minio.use_ssl") config.Configuration.Capacity = viper.GetInt64("capacity") - config.Configuration.MaxFileSize = viper.GetInt64("max_file_size") config.Configuration.DBHost = viper.GetString("db.host") config.Configuration.DBName = viper.GetString("db.name") diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go index 0a1ba18c9..dd0c0205a 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - bconfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/chain" @@ -110,7 +109,6 @@ func init() { if _, err := filestore.SetupFSStoreI(dir+"/tmp", MockFileBlockGetter{}); err != nil { panic(err) } - bconfig.Configuration.MaxFileSize = int64(1 << 30) } func TestBlobberCore_RenameFile(t *testing.T) { diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index eae61c8b8..13b2401be 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -94,7 +94,6 @@ type Config struct { ChallengeMaxRetires int TempFilesCleanupFreq int64 TempFilesCleanupNumWorkers int - MaxFileSize int64 ColdStorageMinimumFileSize int64 ColdStorageTimeLimitInHours int64 diff --git a/code/go/0chain.net/blobbercore/handler/file_command_add.go b/code/go/0chain.net/blobbercore/handler/file_command_add.go index edcf57aa0..85abecb97 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_add.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_add.go @@ -7,7 +7,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -95,10 +94,6 @@ func (cmd *AddFileCommand) ProcessContent(ctx context.Context, req *http.Request allocationSize += fileOutputData.Size } - if allocationSize > config.Configuration.MaxFileSize { - return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") - } - if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize { return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_insert.go b/code/go/0chain.net/blobbercore/handler/file_command_insert.go index 1a815d3d7..16701bf1a 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_insert.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_insert.go @@ -7,7 +7,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -63,11 +62,6 @@ func (cmd *InsertFileCommand) ProcessContent(ctx context.Context, req *http.Requ } defer origfile.Close() - // rejected it on first chunk if actualsize is greater than max_file_size - if config.Configuration.MaxFileSize > 0 && cmd.fileChanger.ActualSize > config.Configuration.MaxFileSize { - return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") - } - fileInputData := &filestore.FileInputData{Name: cmd.fileChanger.Filename, Path: cmd.fileChanger.Path, OnCloud: false} fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) if err != nil { @@ -84,10 +78,6 @@ func (cmd *InsertFileCommand) ProcessContent(ctx context.Context, req *http.Requ if len(cmd.fileChanger.MerkleRoot) > 0 && cmd.fileChanger.MerkleRoot != fileOutputData.MerkleRoot { return result, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") } - //max_file_size = 0 means file size is unlimited - if fileOutputData.Size > config.Configuration.MaxFileSize && config.Configuration.MaxFileSize > 0 { - return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") - } cmd.fileChanger.Hash = fileOutputData.ContentHash cmd.fileChanger.MerkleRoot = fileOutputData.MerkleRoot diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index b92672c5d..f3add4174 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -7,7 +7,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -66,11 +65,6 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ result.Filename = cmd.fileChanger.Filename - // rejected it on first chunk if actualsize is greater than max_file_size - if config.Configuration.MaxFileSize > 0 && cmd.fileChanger.ActualSize > config.Configuration.MaxFileSize { - return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") - } - origfile, _, err := req.FormFile("uploadFile") if err != nil { return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) @@ -116,11 +110,6 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") } - //max_file_size = 0 means file size is unlimited - if fileOutputData.Size > config.Configuration.MaxFileSize && config.Configuration.MaxFileSize > 0 { - return result, common.NewError("file_size_limit_exceeded", "Size for the given file is larger than the max limit") - } - cmd.fileChanger.AllocationID = allocationObj.ID cmd.fileChanger.Size = allocationSize diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 1375092cd..bd5a2cb0f 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -29,7 +29,6 @@ import ( "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - bconfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -85,7 +84,6 @@ func init() { if _, err := filestore.SetupFSStoreI(dir+"/tmp", MockFileBlockGetter{}); err != nil { panic(err) } - bconfig.Configuration.MaxFileSize = int64(1 << 30) } func setup(t *testing.T) { diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index d741b8d28..94a9b6192 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -37,8 +37,6 @@ challenge_completion_time: 2m # duration to complete a challenge read_lock_timeout: 1m write_lock_timeout: 1m -max_file_size: 104857600 #10MB,please set it with 0 if file size is unlimited - # update_allocations_interval used to refresh known allocation objects from SC update_allocations_interval: 1m From 9b375a5d119ea9789fea5661b44e274e92d8b40a Mon Sep 17 00:00:00 2001 From: Andika Demas Riyandi Date: Fri, 10 Dec 2021 23:44:45 +0700 Subject: [PATCH 072/161] ignore token that never expired (has 0 value) (#461) * ignore token that never expired (has 0 value) * grouping expiration check Co-authored-by: Lz * improved expiration checking Co-authored-by: Lz --- code/go/0chain.net/blobbercore/readmarker/entity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/readmarker/entity.go b/code/go/0chain.net/blobbercore/readmarker/entity.go index 3245c5eac..738bdaa69 100644 --- a/code/go/0chain.net/blobbercore/readmarker/entity.go +++ b/code/go/0chain.net/blobbercore/readmarker/entity.go @@ -43,7 +43,7 @@ func (authToken *AuthTicket) Verify(allocationObj *allocation.Allocation, client if authToken.ClientID != clientID && len(authToken.ClientID) > 0 { return common.NewError("invalid_parameters", "Invalid auth ticket. Client ID mismatch") } - if authToken.Expiration < authToken.Timestamp || authToken.Expiration < common.Now() { +if authToken.Expiration > 0 && (authToken.Expiration < authToken.Timestamp || authToken.Expiration < common.Now() ) { return common.NewError("invalid_parameters", "Invalid auth ticket. Expired ticket") } From 1d86b6a12f53a81b579d3ae7b7f0831fe3de5ecf Mon Sep 17 00:00:00 2001 From: Ryan Date: Sat, 11 Dec 2021 21:37:29 +0000 Subject: [PATCH 073/161] Feature/use system tests action (#463) * replace existing system tests trigger and poll with native action * read correct var * add substitution correctly * allow tests to be skipped * re-order params to push blobber/validator options first * test commit * fic formatting * retry failures * add auto system tests * ignore validator publish which is failing * add self hosted runner for system tests * hardcode a runner that works * hardcode a runner that works * hardcode a runner that works * remove conditional * change runners again!!! * debug * test manual trigger * test manual trigger * add manual PR notification to automatic workflow * derive docker image from branch * derive docker image from branch * new commit * debug logging * fix logic to support merge PRs * add escape for when PR not raised * don't run system tests on workflow dispatch for dockerize * don't run system tests on workflow dispatch for dockerize * add auto cancel on new commit so resources are not hogged on system tests * add workflow dispatch skip to default system tests instead of manual * rename workflow to be consistent with other repos * only check image presence if not skipping tests --- .../build-&-publish-docker-image.yml | 105 +++++++++++++++++- .github/workflows/system_tests.yml | 82 ++++++++++---- .../blobbercore/readmarker/entity.go | 2 +- 3 files changed, 163 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 65c6ed0fb..0e865ff5a 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -1,4 +1,8 @@ -name: Dockerize +name: "Build & Publish Docker Image" + +concurrency: + group: "${{ github.ref }}" + cancel-in-progress: true on: push: @@ -42,7 +46,7 @@ jobs: build_blobber: needs: build_base - runs-on: [self-hosted, build] + runs-on: [self-hosted, conductor-test] steps: - name: Set GITHUB_ENV run: | @@ -79,13 +83,13 @@ jobs: publish_blobber: needs: build_blobber - runs-on: [self-hosted] + runs-on: [self-hosted, conductor-test] steps: - name: Set GITHUB_ENV run: | echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - + - name: Download blobber uses: actions/download-artifact@v2 with: @@ -114,7 +118,7 @@ jobs: build_validator: needs: build_base - runs-on: [self-hosted, build] + runs-on: [self-hosted, conductor-test] steps: - name: Set GITHUB_ENV run: | @@ -153,7 +157,7 @@ jobs: publish_validator: needs: build_validator - runs-on: [self-hosted] + runs-on: [self-hosted, conductor-test] steps: - name: Set GITHUB_ENV run: | @@ -183,3 +187,92 @@ jobs: SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA + + system-tests: + if: github.event_name != 'workflow_dispatch' + needs: [publish_blobber, publish_validator] + runs-on: [ tests-suite ] + steps: + - name: "Get current PR" + uses: jwalton/gh-find-current-pr@v1 + id: findPr + with: + github-token: ${{ github.token }} + + - name: "Set PR Status Pending" + uses: niteoweb/pull_request_status_action@v1.0.0 + if: steps.findPr.outputs.number + with: + pr_number: ${{ steps.findPr.outputs.pr }} + description: "System tests running with default config..." + state: "pending" + repository: ${{ github.repository }} + context: "0Chain System Tests" + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: "Setup" + run: | + echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> $GITHUB_ENV + echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV + echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV + + - name: "Deploy 0Chain" + uses: 0chain/actions/deploy-0chain@master + with: + kube_config: ${{ secrets[format('DEV{0}KC', env.RUNNER_NUMBER)] }} + teardown_condition: "TESTS_PASSED" + miner_image: staging + sharder_image: staging + blobber_image: ${{ env.TAG }}-${{ env.SHORT_SHA }} + validator_image: ${{ env.TAG }}-${{ env.SHORT_SHA }} + zbox_image: staging + zblock_image: staging + zdns_image: staging + explorer_image: latest + zproxy_image: staging + zsearch_image: staging + blobber_stake_image: latest + + - name: "Run System tests" + uses: 0chain/actions/run-system-tests@master + with: + system_tests_branch: master + network: ${{ env.NETWORK_URL }} + zbox_cli_branch: staging + zwallet_cli_branch: staging + smart_contract_owner_wallet_json: ${{ secrets.SMART_CONTRACT_OWNER_WALLET_JSON }} + svc_account_secret: ${{ github.token }} + deploy_report_page: false + archive_results: true + run_flaky_tests: false + retry_failures: true + + - name: "Set PR Status Success" + if: ${{ success() && steps.findPr.outputs.number }} + uses: niteoweb/pull_request_status_action@v1.0.0 + with: + pr_number: ${{ steps.findPr.outputs.pr }} + description: "System tests with default config completed successfully" + state: ${{ job.status }} + repository: ${{ github.repository }} + context: "0Chain System Tests" + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: "Set PR Status Failed" + if: ${{ failure() && steps.findPr.outputs.number }} + uses: niteoweb/pull_request_status_action@v1.0.0 + with: + pr_number: ${{ steps.findPr.outputs.pr }} + description: "System tests with default config failed. Try running manually with more specific config" + state: ${{ job.status }} + repository: ${{ github.repository }} + context: "0Chain System Tests" + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ github.token }} \ No newline at end of file diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 299c7157b..378b34c45 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -11,14 +11,6 @@ on: description: 'system_tests branch. Containing the tests you wish to run' default: 'master' required: true - blobber_image: - description: 'blobber DOCKER IMAGE to deploy' - default: 'staging' - required: true - validator_image: - description: 'validator DOCKER IMAGE to deploy' - default: 'staging' - required: true zbox_cli_branch: description: '0Box CLI (branch or commit SHA) which the tests will use' default: 'staging' @@ -50,29 +42,81 @@ on: jobs: system-tests: runs-on: [ tests-suite ] - timeout-minutes: 40 + timeout-minutes: 90 steps: - name: "Get current PR" uses: jwalton/gh-find-current-pr@v1 id: findPr with: - github-token: ${{ github.token }} - + github-token: ${{ github.token }} + - name: "Setup Test Run" run: | + echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> $GITHUB_ENV + echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV + + - name: "Check blobber image has been published" + if: ${{ github.event.inputs.skip_tests == 'FALSE' }} + run: | + exit_code=0 + echo "Checking blobber image [${{ env.TAG }}-${{ env.SHORT_SHA }}] is present..." + docker pull 0chaindev/blobber:${{ env.TAG }}-${{ env.SHORT_SHA }} || exit_code=$? + + if [[ $exit_code == 0 ]]; + then + echo "blobber docker blobber image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" + echo "BLOBBER_IMAGE=${{ env.TAG }}-${{ env.SHORT_SHA }}" >> $GITHUB_ENV + else + if [ -z "${{ steps.findPr.outputs.pr }}" ]; + then + echo "blobber docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}] and PR not raised so will not check for PR merge image. Please publish image manually." && exit 1; + fi + echo "blobber docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" + git fetch origin pull/${{ steps.findPr.outputs.pr }}/merge:merge && git checkout merge + MERGE_SHORT_SHA=$(git rev-parse HEAD | head -c 8) + echo "Checking blobber image [merge-$MERGE_SHORT_SHA] is present..." + docker pull 0chaindev/blobber:merge-$MERGE_SHORT_SHA || { echo "Docker image for blobber could not be found for either branch commit sha [${{ env.SHORT_SHA }}] or PR merge commit sha [$MERGE_SHORT_SHA]. Please publish image manually and try again." && exit 1; } + echo "blobber docker image found for [merge-$MERGE_SHORT_SHA]" + echo "BLOBBER_IMAGE=merge-$MERGE_SHORT_SHA" >> $GITHUB_ENV + fi + + - name: "Check validator image has been published" + if: ${{ github.event.inputs.skip_tests == 'FALSE' }} + run: | + exit_code=0 + echo "Checking valiator image [${{ env.TAG }}-${{ env.SHORT_SHA }}] is present..." + docker pull 0chaindev/validator:${{ env.TAG }}-${{ env.SHORT_SHA }} || exit_code=$? + + if [[ $exit_code == 0 ]]; + then + echo "valiator docker valiator image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" + echo "VALIDATOR_IMAGE=${{ env.TAG }}-${{ env.SHORT_SHA }}" >> $GITHUB_ENV + else + if [ -z "${{ steps.findPr.outputs.pr }}" ]; + then + echo "valiator docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}] and PR not raised so will not check for PR merge image. Please publish image manually." && exit 1; + fi + echo "valiator docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" + git fetch origin pull/${{ steps.findPr.outputs.pr }}/merge:merge && git checkout merge + MERGE_SHORT_SHA=$(git rev-parse HEAD | head -c 8) + echo "Checking valiator image [merge-$MERGE_SHORT_SHA] is present..." + docker pull 0chaindev/validator:merge-$MERGE_SHORT_SHA || { echo "Docker image for valiator could not be found for either branch commit sha [${{ env.SHORT_SHA }}] or PR merge sha [$MERGE_SHORT_SHA]. Please publish image manually and try again." && exit 1; } + echo "valiator docker image found for [merge-$MERGE_SHORT_SHA]" + echo "VALIDATOR_IMAGE=merge-$MERGE_SHORT_SHA" >> $GITHUB_ENV + fi - name: "Set PR Status Pending" uses: niteoweb/pull_request_status_action@v1.0.0 if: steps.findPr.outputs.number with: pr_number: ${{ steps.findPr.outputs.pr }} - description: "System tests running" + description: "Manual system tests running" state: "pending" repository: ${{ github.repository }} - context: ${{ github.workflow }} + context: "0Chain System Tests" target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} env: GITHUB_TOKEN: ${{ github.token }} @@ -85,8 +129,8 @@ jobs: teardown_condition: "TESTS_PASSED" miner_image: ${{ github.event.inputs.miner_image }} sharder_image: ${{ github.event.inputs.sharder_image }} - blobber_image: ${{ github.event.inputs.blobber_image }} - validator_image: ${{ github.event.inputs.validator_image }} + blobber_image: ${{ env.BLOBBER_IMAGE }} + validator_image: ${{ env.VALIDATOR_IMAGE }} zbox_image: ${{ github.event.inputs.zbox_image }} zblock_image: ${{ github.event.inputs.zblock_image }} zdns_image: staging @@ -115,10 +159,10 @@ jobs: uses: niteoweb/pull_request_status_action@v1.0.0 with: pr_number: ${{ steps.findPr.outputs.pr }} - description: "System tests success" + description: "Manual system tests success" state: ${{ job.status }} repository: ${{ github.repository }} - context: ${{ github.workflow }} + context: "0Chain System Tests" target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} env: GITHUB_TOKEN: ${{ github.token }} @@ -128,10 +172,10 @@ jobs: uses: niteoweb/pull_request_status_action@v1.0.0 with: pr_number: ${{ steps.findPr.outputs.pr }} - description: "System tests failure" + description: "Manual system tests failure" state: ${{ job.status }} repository: ${{ github.repository }} - context: ${{ github.workflow }} + context: "0Chain System Tests" target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} env: GITHUB_TOKEN: ${{ github.token }} diff --git a/code/go/0chain.net/blobbercore/readmarker/entity.go b/code/go/0chain.net/blobbercore/readmarker/entity.go index 738bdaa69..8f3d516ec 100644 --- a/code/go/0chain.net/blobbercore/readmarker/entity.go +++ b/code/go/0chain.net/blobbercore/readmarker/entity.go @@ -43,7 +43,7 @@ func (authToken *AuthTicket) Verify(allocationObj *allocation.Allocation, client if authToken.ClientID != clientID && len(authToken.ClientID) > 0 { return common.NewError("invalid_parameters", "Invalid auth ticket. Client ID mismatch") } -if authToken.Expiration > 0 && (authToken.Expiration < authToken.Timestamp || authToken.Expiration < common.Now() ) { + if authToken.Expiration > 0 && (authToken.Expiration < authToken.Timestamp || authToken.Expiration < common.Now()) { return common.NewError("invalid_parameters", "Invalid auth ticket. Expired ticket") } From 82d9f6bd296374758652747ae23703a5ca6edb7f Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 13 Dec 2021 16:55:12 +0000 Subject: [PATCH 074/161] Update PR template and typo fixes (#466) * replace existing system tests trigger and poll with native action * read correct var * add substitution correctly * allow tests to be skipped * re-order params to push blobber/validator options first * test commit * fic formatting * retry failures * add auto system tests * ignore validator publish which is failing * add self hosted runner for system tests * hardcode a runner that works * hardcode a runner that works * hardcode a runner that works * remove conditional * change runners again!!! * debug * test manual trigger * test manual trigger * add manual PR notification to automatic workflow * derive docker image from branch * derive docker image from branch * new commit * debug logging * fix logic to support merge PRs * add escape for when PR not raised * don't run system tests on workflow dispatch for dockerize * don't run system tests on workflow dispatch for dockerize * add auto cancel on new commit so resources are not hogged on system tests * add workflow dispatch skip to default system tests instead of manual * rename workflow to be consistent with other repos * only check image presence if not skipping tests * Add PR template --- .github/workflows/system_tests.yml | 4 ++-- PULL_REQUEST_TEMPLATE.md | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 PULL_REQUEST_TEMPLATE.md diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 378b34c45..6682afff5 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -67,7 +67,7 @@ jobs: if [[ $exit_code == 0 ]]; then - echo "blobber docker blobber image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" + echo "blobber docker image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" echo "BLOBBER_IMAGE=${{ env.TAG }}-${{ env.SHORT_SHA }}" >> $GITHUB_ENV else if [ -z "${{ steps.findPr.outputs.pr }}" ]; @@ -92,7 +92,7 @@ jobs: if [[ $exit_code == 0 ]]; then - echo "valiator docker valiator image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" + echo "valiator docker image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" echo "VALIDATOR_IMAGE=${{ env.TAG }}-${{ env.SHORT_SHA }}" >> $GITHUB_ENV else if [ -z "${{ steps.findPr.outputs.pr }}" ]; diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..5af67e46d --- /dev/null +++ b/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,3 @@ +Tasks to complete before merging PR: +- [ ] Ensure system tests are passing. If not [Run them manually](https://github.com/0chain/blobber/actions/workflows/system_tests.yml) to check for any regressions :clipboard: +- [ ] Do any new system tests need added to test this change? do any existing system tests need updated? If so create a branch at [0chain/system_test](https://github.com/0chain/system_test) \ No newline at end of file From 9631e8f0cd51f43112f7a7949a31ff4d02ab3cb7 Mon Sep 17 00:00:00 2001 From: stewartie4 Date: Thu, 16 Dec 2021 16:45:59 +0000 Subject: [PATCH 075/161] checkout current branch explicitly on manual system tests --- .github/workflows/system_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 6682afff5..48243a585 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -44,7 +44,9 @@ jobs: runs-on: [ tests-suite ] timeout-minutes: 90 steps: - + + - uses: actions/checkout@v2 + - name: "Get current PR" uses: jwalton/gh-find-current-pr@v1 id: findPr From 8866d995b18da23c909ecbafb3d675981be988ca Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 17 Dec 2021 01:34:37 +0000 Subject: [PATCH 076/161] checkout current branch explicitly on manual system tests (#469) * checkout current branch explicitly on manual system tests * remove additional slow running stages + add resiliency on publish * remove additional slow running stages + add resiliency on publish * rename stages --- .../build-&-publish-docker-image.yml | 132 ++---------------- .github/workflows/system_tests.yml | 2 +- 2 files changed, 15 insertions(+), 119 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 0e865ff5a..110f9fe95 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -20,33 +20,8 @@ env: VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} jobs: - build_base: + build_and_publish_blobber: runs-on: [self-hosted, build] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - name: Clone blobber - uses: actions/checkout@v1 - - - name: Build blobber_base - run: ./docker.local/bin/build.base.sh - - - name: Export blobber_base - run: | - mkdir -p /tmp/0chain/ - docker save "blobber_base" > /tmp/0chain/blobber_base.tar - - - name: Upload blobber_base - uses: actions/upload-artifact@v2 - with: - name: blobber_base - path: /tmp/0chain/blobber_base.tar - - build_blobber: - needs: build_base - runs-on: [self-hosted, conductor-test] steps: - name: Set GITHUB_ENV run: | @@ -56,69 +31,29 @@ jobs: - name: Clone blobber uses: actions/checkout@v1 - - - name: Download blobber_base - uses: actions/download-artifact@v2 - with: - name: blobber_base - path: /tmp/0chain - - - name: Load blobber_base - run: | - docker load --input /tmp/0chain/blobber_base.tar + - name: Build blobber_base + run: ./docker.local/bin/build.base.sh - name: Build blobber run: ./docker.local/bin/build.blobber.sh - - - name: Export blobber - run: | - mkdir -p /tmp/0chain/ - docker save "blobber" > /tmp/0chain/blobber.tar - - - name: Upload blobber - uses: actions/upload-artifact@v2 - with: - name: blobber - path: /tmp/0chain/blobber.tar - - publish_blobber: - needs: build_blobber - runs-on: [self-hosted, conductor-test] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - - name: Download blobber - uses: actions/download-artifact@v2 - with: - name: blobber - path: /tmp/0chain - - - name: Load blobber - run: | - docker load --input /tmp/0chain/blobber.tar - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push blobber run: | docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG - docker push ${BLOBBER_REGISTRY}:$TAG + docker push ${BLOBBER_REGISTRY}:$TAG || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG; } SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA - docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA - - + docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA; } - build_validator: - needs: build_base - runs-on: [self-hosted, conductor-test] + build_and_publish_validator: + runs-on: [self-hosted, build] steps: - name: Set GITHUB_ENV run: | @@ -128,69 +63,30 @@ jobs: - name: Clone blobber uses: actions/checkout@v1 - - - name: Download blobber_base - uses: actions/download-artifact@v2 - with: - name: blobber_base - path: /tmp/0chain - - - name: Load blobber_base - run: | - docker load --input /tmp/0chain/blobber_base.tar - - #docker image ls -a + - name: Build blobber_base + run: ./docker.local/bin/build.base.sh - name: Build validator run: ./docker.local/bin/build.validator.sh - - - name: Export validator - run: | - mkdir -p /tmp/0chain/ - docker save "validator" > /tmp/0chain/validator.tar - - - name: Upload validator - uses: actions/upload-artifact@v2 - with: - name: validator - path: /tmp/0chain/validator.tar - - publish_validator: - needs: build_validator - runs-on: [self-hosted, conductor-test] - steps: - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - - name: Download validator - uses: actions/download-artifact@v2 - with: - name: validator - path: /tmp/0chain - - - name: Load validator - run: | - docker load --input /tmp/0chain/validator.tar - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push validator run: | docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG - docker push ${VALIDATOR_REGISTRY}:$TAG + docker push ${VALIDATOR_REGISTRY}:$TAG || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG; } SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA - docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA + docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA; } system-tests: if: github.event_name != 'workflow_dispatch' - needs: [publish_blobber, publish_validator] + needs: [build_and_publish_blobber, build_and_publish_validator] runs-on: [ tests-suite ] steps: - name: "Get current PR" diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 48243a585..049e840a2 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v2 - + - name: "Get current PR" uses: jwalton/gh-find-current-pr@v1 id: findPr From e30ded366c95aaf7cd1d15e4709a5e9ac9b75707 Mon Sep 17 00:00:00 2001 From: Rezoan Tamal <36768796+iamrz1@users.noreply.github.com> Date: Tue, 28 Dec 2021 21:37:34 +0600 Subject: [PATCH 077/161] Fix stats error for non-html endpoints (#475) * resolve stats pagination error by setting request data in context * handle nil value for pagination request data for endpoints without pagintion --- .../blobbercore/stats/blobberstats.go | 34 +++++++++++-------- .../blobbercore/stats/pagination.go | 26 +++++++------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index 73ef155d2..831841c6f 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -78,8 +78,8 @@ type BlobberStats struct { InfraStats InfraStats `json:"-"` DBStats *DBStats `json:"-"` FailedChallengeList []ChallengeEntity `json:"-"` - FailedChallengePagination Pagination `json:"-"` - AllocationListPagination Pagination `json:"-"` + FailedChallengePagination *Pagination `json:"failed_challenge_pagination,omitempty"` + AllocationListPagination *Pagination `json:"allocation_list_pagination,omitempty"` // configurations Capacity int64 `json:"capacity"` @@ -182,7 +182,6 @@ func (bs *BlobberStats) loadDetailedStats(ctx context.Context) { func (bs *BlobberStats) loadInfraStats(ctx context.Context) { healthIn := ctx.Value(HealthDataKey) if healthIn == nil { - Logger.Error("loadInfraStats err where health value nil") return } health := healthIn.(string) @@ -215,7 +214,6 @@ func (bs *BlobberStats) loadDBStats() { func (bs *BlobberStats) loadFailedChallengeList(ctx context.Context) { fcrdI := ctx.Value(FailedChallengeRequestDataKey) if fcrdI == nil { - Logger.Error("fcrd should not be nil") return } fcrd := fcrdI.(RequestData) @@ -228,7 +226,7 @@ func (bs *BlobberStats) loadFailedChallengeList(ctx context.Context) { bs.FailedChallengeList = fcs pagination := GeneratePagination(fcrd.Page, fcrd.Limit, fcrd.Offset, count) - bs.FailedChallengePagination = *pagination + bs.FailedChallengePagination = pagination } func (bs *BlobberStats) loadStats(ctx context.Context) { @@ -297,19 +295,25 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { bs.AllocationStats = make([]*AllocationStats, 0) var ( - db = datastore.GetStore().GetTransaction(ctx) - rows *sql.Rows - err error + db = datastore.GetStore().GetTransaction(ctx) + rows *sql.Rows + err error + requestData *RequestData + offset, limit int ) alrdI := ctx.Value(AllocationListRequestDataKey) if alrdI == nil { - Logger.Error("loadAllocationStats err: alrd should not be nil", zap.Any("err", err)) - return + offset = 0 + limit = 20 + } else { + alrd := alrdI.(RequestData) + requestData = &alrd + offset = requestData.Offset + limit = requestData.Limit } - alrd := alrdI.(RequestData) - rows, err = db.Table("reference_objects").Offset(alrd.Offset).Limit(alrd.Limit). + rows, err = db.Table("reference_objects").Offset(offset).Limit(limit). Select(` reference_objects.allocation_id, SUM(reference_objects.size) as files_size, @@ -365,8 +369,10 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { return } - pagination := GeneratePagination(alrd.Page, alrd.Limit, alrd.Offset, int(count)) - bs.AllocationListPagination = *pagination + if requestData != nil { + pagination := GeneratePagination(requestData.Page, requestData.Limit, requestData.Offset, int(count)) + bs.AllocationListPagination = pagination + } } func (bs *BlobberStats) loadChallengeStats(ctx context.Context) { diff --git a/code/go/0chain.net/blobbercore/stats/pagination.go b/code/go/0chain.net/blobbercore/stats/pagination.go index 69c9f4cf7..fce3cadcf 100644 --- a/code/go/0chain.net/blobbercore/stats/pagination.go +++ b/code/go/0chain.net/blobbercore/stats/pagination.go @@ -11,19 +11,19 @@ type RequestData struct { } type Pagination struct { - TotalItems int - CurrentPage int - Limit int - TotalPages int - Offset int - PrevPage int - NextPage int - HasPrev bool - HasNext bool - HasDivider bool - CurrentPageIsInDivider bool - FirstFew []int - LastFew []int + TotalItems int `json:"total_items"` + CurrentPage int `json:"current_page"` + Limit int `json:"limit"` + TotalPages int `json:"total_pages"` + Offset int `json:"offset"` + PrevPage int `json:"prev_page"` + NextPage int `json:"next_page"` + HasPrev bool `json:"-"` + HasNext bool `json:"-"` + HasDivider bool `json:"-"` + CurrentPageIsInDivider bool `json:"-"` + FirstFew []int `json:"-"` + LastFew []int `json:"-"` } type PageLimitOffsetData struct { From 9abf9e102af2b033cf146db1d8da64afc69ec82a Mon Sep 17 00:00:00 2001 From: prince Date: Wed, 29 Dec 2021 13:46:07 +0530 Subject: [PATCH 078/161] Bug/issue 401 copy file when no dir (#464) * Create dir logic added if given path is not available. Signed-off-by: Pradip Parmar * copy_ok testcase query mock change. Signed-off-by: Pradip Parmar * removed createdir methor from obejct operation and added mkdir in ref. Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * testcase fix Signed-off-by: Pradip Parmar * allocation id issue resolved Signed-off-by: Pradip Parmar * fix(delete):#451 improved delete logic on reference_objects * handling for revert on fail for copy command. Signed-off-by: Pradip Parmar * testcases updated for copy_ok. Signed-off-by: Pradip Parmar * trying system test with copy-zbox branch. Signed-off-by: Pradip Parmar * Revert "fix(delete):#451 improved delete logic on reference_objects" This reverts commit 29c5f5227b882e76416500305f8f01977ec9a0a4. * handling for copy to root Signed-off-by: Pradip Parmar * updated testcase branch to validate testcase changes. Signed-off-by: Pradip Parmar * removed testing branch from system-test Signed-off-by: Pradip Parmar * system testcase branch name change Signed-off-by: Pradip Parmar * wallet id reverted Signed-off-by: Pradip Parmar * copy in dir with child fixed Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Lz --- .../blobbercore/allocation/copyfilechange.go | 29 ++++++++++++------- .../blobbercore/handler/handler_test.go | 14 --------- .../handler/object_operation_handler.go | 5 ---- .../0chain.net/blobbercore/reference/ref.go | 4 +-- .../blobbercore/reference/referencepath.go | 2 ++ 5 files changed, 21 insertions(+), 33 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index d655633fa..8ba79f13a 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -27,18 +27,24 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC if err != nil { return nil, err } - destRef, err := reference.GetRefWithSortedChildren(ctx, rf.AllocationID, rf.DestPath) - if err != nil || destRef.Type != reference.DIRECTORY { - return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") - } - rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot) + if rf.DestPath == "/" { + destRef, err := reference.GetRefWithSortedChildren(ctx, rf.AllocationID, rf.DestPath) + if err != nil || destRef.Type != reference.DIRECTORY { + return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") + } + rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot) - if destRef.ParentPath == "" { _, err = destRef.CalculateHash(ctx, true) return destRef, err } + // it will create new dir if it is not available in db + destRef, err := reference.Mkdir(ctx, rf.AllocationID, rf.DestPath) + if err != nil || destRef.Type != reference.DIRECTORY { + return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") + } + path, _ := filepath.Split(rf.DestPath) path = filepath.Clean(path) tSubDirs := reference.GetSubDirsFromPath(path) @@ -67,20 +73,21 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC return nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") } } - var foundRef *reference.Ref = nil + childIndex := -1 for i, child := range dirRef.Children { if child.Path == rf.DestPath && child.Type == reference.DIRECTORY { - foundRef = dirRef.Children[i] - dirRef.RemoveChild(i) - dirRef.AddChild(destRef) + childIndex = i break } } - if foundRef == nil { + if childIndex == -1 { return nil, common.NewError("file_not_found", "Destination Object to copy to not found in blobber") } + foundRef := dirRef.Children[childIndex] + rf.processCopyRefs(ctx, affectedRef, foundRef, allocationRoot) + _, err = rootRef.CalculateHash(ctx, true) return rootRef, err diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index bd5a2cb0f..bc7b8f1e5 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -898,20 +898,6 @@ func TestHandlers_Requiring_Signature(t *testing.T) { WithArgs(aa, aa). WillReturnError(errors.New("")) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(aa, aa). - WillReturnRows( - sqlmock.NewRows([]string{"type"}). - AddRow(reference.DIRECTORY), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(aa, aa). - WillReturnRows( - sqlmock.NewRows([]string{"type"}). - AddRow(reference.DIRECTORY), - ) - mock.ExpectExec(`INSERT INTO "allocation_connections"`). WithArgs(aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 4c008ae94..ad7503165 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -934,11 +934,6 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") } - destRef, err = reference.Mkdir(ctx, allocationID, destPath) - if err != nil || destRef.Type != reference.DIRECTORY { - return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") - } - allocationChange := &allocation.AllocationChange{} allocationChange.ConnectionID = connectionObj.ConnectionID allocationChange.Size = objectRef.Size diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 03e238ad0..013a88cf9 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -172,7 +172,7 @@ func (r *Ref) SetAttributes(attr *Attributes) (err error) { return } -// Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be returned +// Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child func Mkdir(ctx context.Context, allocationID string, destpath string) (*Ref, error) { var dirRef *Ref db := datastore.GetStore().GetTransaction(ctx) @@ -184,7 +184,6 @@ func Mkdir(ctx context.Context, allocationID string, destpath string) (*Ref, err currentPath := filepath.Join("/", filepath.Join(dirs[:i+1]...)) ref, err := GetReference(ctx, allocationID, currentPath) if err == nil { - ref.AddChild(dirRef) dirRef = ref continue } @@ -208,7 +207,6 @@ func Mkdir(ctx context.Context, allocationID string, destpath string) (*Ref, err return nil, err } - newRef.AddChild(dirRef) dirRef = newRef } diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 2d3e84b5b..1e1580dd4 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -4,6 +4,7 @@ import ( "context" "math" "path/filepath" + "strings" "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" @@ -27,6 +28,7 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths [ db := datastore.GetStore().GetTransaction(ctx) pathsAdded := make(map[string]bool) for _, path := range paths { + path = strings.TrimSuffix(path, "/") if _, ok := pathsAdded[path]; !ok { db = db.Where(Ref{ParentPath: path, AllocationID: allocationID}) pathsAdded[path] = true From 1dc97c9b8d5cdc227d34ada9e7b1a0ab3bdcce0b Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 30 Dec 2021 00:42:44 +0000 Subject: [PATCH 079/161] notify all PRs on manual system tests (#478) --- .../build-&-publish-docker-image.yml | 43 ++-- .github/workflows/system_tests.yml | 210 ++++-------------- .github/workflows/tests.yml | 10 +- PULL_REQUEST_TEMPLATE.md | 6 +- 4 files changed, 77 insertions(+), 192 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 110f9fe95..a68592334 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -1,7 +1,7 @@ name: "Build & Publish Docker Image" concurrency: - group: "${{ github.ref }}" + group: "publish-${{ github.ref }}" cancel-in-progress: true on: @@ -25,8 +25,9 @@ jobs: steps: - name: Set GITHUB_ENV run: | - echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV + echo "TAG=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF) | sed 's/\//-/g')" >> $GITHUB_ENV + echo "SHA=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}')" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 @@ -48,7 +49,7 @@ jobs: docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG docker push ${BLOBBER_REGISTRY}:$TAG || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG; } - SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) + SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA; } @@ -57,8 +58,9 @@ jobs: steps: - name: Set GITHUB_ENV run: | - echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV + echo "TAG=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF) | sed 's/\//-/g')" >> $GITHUB_ENV + echo "SHA=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}')" >> $GITHUB_ENV - name: Clone blobber uses: actions/checkout@v1 @@ -80,7 +82,7 @@ jobs: docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG docker push ${VALIDATOR_REGISTRY}:$TAG || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG; } - SHORT_SHA=$(echo $GITHUB_SHA | head -c 8) + SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA; } @@ -95,7 +97,7 @@ jobs: with: github-token: ${{ github.token }} - - name: "Set PR Status Pending" + - name: "Set PR status as pending" uses: niteoweb/pull_request_status_action@v1.0.0 if: steps.findPr.outputs.number with: @@ -110,9 +112,9 @@ jobs: - name: "Setup" run: | - echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> $GITHUB_ENV - echo "BRANCH=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV + echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV + echo "SHORT_SHA=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}') | head -c 8)" >> $GITHUB_ENV + echo "TAG=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF) | sed 's/\//-/g')" >> $GITHUB_ENV echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV @@ -147,25 +149,12 @@ jobs: run_flaky_tests: false retry_failures: true - - name: "Set PR Status Success" - if: ${{ success() && steps.findPr.outputs.number }} + - name: "Set PR status as ${{ job.status }}" + if: ${{ (success() || failure()) && steps.findPr.outputs.number }} uses: niteoweb/pull_request_status_action@v1.0.0 with: pr_number: ${{ steps.findPr.outputs.pr }} - description: "System tests with default config completed successfully" - state: ${{ job.status }} - repository: ${{ github.repository }} - context: "0Chain System Tests" - target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: "Set PR Status Failed" - if: ${{ failure() && steps.findPr.outputs.number }} - uses: niteoweb/pull_request_status_action@v1.0.0 - with: - pr_number: ${{ steps.findPr.outputs.pr }} - description: "System tests with default config failed. Try running manually with more specific config" + description: "System tests with default config ${{ job.status }}" state: ${{ job.status }} repository: ${{ github.repository }} context: "0Chain System Tests" diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 049e840a2..b4a53e9fd 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -1,183 +1,71 @@ -name: "0Chain System Tests" - -concurrency: - group: "${{ github.ref }}" - cancel-in-progress: true +name: "0Chain System Tests" on: workflow_dispatch: inputs: - system_tests_branch: - description: 'system_tests branch. Containing the tests you wish to run' + system_tests_branch: + description: 'system_tests branch containing the tests you wish to run' default: 'master' required: true - zbox_cli_branch: - description: '0Box CLI (branch or commit SHA) which the tests will use' + miner_branch: + description: 'miner branch to deploy' + default: 'staging' + required: true + sharder_branch: + description: 'sharder branch to deploy' + default: 'staging' + required: true + zbox_cli_branch: + description: '0Box CLI branch which the tests will use' default: 'staging' required: true - zwallet_cli_branch: - description: '0Wallet CLI (branch or commit SHA) which the tests will use' + zwallet_cli_branch: + description: '0Wallet CLI branch which the tests will use' default: 'staging' required: true - miner_image: - description: 'miner DOCKER IMAGE to deploy' + zbox_branch: + description: '0box branch to deploy' default: 'staging' required: true - sharder_image: - description: 'sharder DOCKER IMAGE to deploy' + zblock_branch: + description: '0block branch to deploy' default: 'staging' required: true - zbox_image: - description: '0box DOCKER IMAGE to deploy' + zdns_branch: + description: '0dns branch to deploy' default: 'staging' required: true - zblock_image: - description: '0block DOCKER IMAGE to deploy' + explorer_branch: + description: 'explorer branch to deploy' default: 'staging' required: true - skip_tests: + skip_tests: description: '(LAST RESORT ONLY) Skip system tests. This will allow a PR to merge without requiring a green test run. *By using you certify that the code being merged is not causing system tests to fail*' default: 'FALSE' required: true jobs: - system-tests: - runs-on: [ tests-suite ] - timeout-minutes: 90 - steps: - - - uses: actions/checkout@v2 - - - name: "Get current PR" - uses: jwalton/gh-find-current-pr@v1 - id: findPr - with: - github-token: ${{ github.token }} - - - name: "Setup Test Run" - run: | - echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g' )" >> $GITHUB_ENV - echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV - echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV - - - name: "Check blobber image has been published" - if: ${{ github.event.inputs.skip_tests == 'FALSE' }} - run: | - exit_code=0 - echo "Checking blobber image [${{ env.TAG }}-${{ env.SHORT_SHA }}] is present..." - docker pull 0chaindev/blobber:${{ env.TAG }}-${{ env.SHORT_SHA }} || exit_code=$? - - if [[ $exit_code == 0 ]]; - then - echo "blobber docker image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" - echo "BLOBBER_IMAGE=${{ env.TAG }}-${{ env.SHORT_SHA }}" >> $GITHUB_ENV - else - if [ -z "${{ steps.findPr.outputs.pr }}" ]; - then - echo "blobber docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}] and PR not raised so will not check for PR merge image. Please publish image manually." && exit 1; - fi - echo "blobber docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" - git fetch origin pull/${{ steps.findPr.outputs.pr }}/merge:merge && git checkout merge - MERGE_SHORT_SHA=$(git rev-parse HEAD | head -c 8) - echo "Checking blobber image [merge-$MERGE_SHORT_SHA] is present..." - docker pull 0chaindev/blobber:merge-$MERGE_SHORT_SHA || { echo "Docker image for blobber could not be found for either branch commit sha [${{ env.SHORT_SHA }}] or PR merge commit sha [$MERGE_SHORT_SHA]. Please publish image manually and try again." && exit 1; } - echo "blobber docker image found for [merge-$MERGE_SHORT_SHA]" - echo "BLOBBER_IMAGE=merge-$MERGE_SHORT_SHA" >> $GITHUB_ENV - fi - - - name: "Check validator image has been published" - if: ${{ github.event.inputs.skip_tests == 'FALSE' }} - run: | - exit_code=0 - echo "Checking valiator image [${{ env.TAG }}-${{ env.SHORT_SHA }}] is present..." - docker pull 0chaindev/validator:${{ env.TAG }}-${{ env.SHORT_SHA }} || exit_code=$? - - if [[ $exit_code == 0 ]]; - then - echo "valiator docker image found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" - echo "VALIDATOR_IMAGE=${{ env.TAG }}-${{ env.SHORT_SHA }}" >> $GITHUB_ENV - else - if [ -z "${{ steps.findPr.outputs.pr }}" ]; - then - echo "valiator docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}] and PR not raised so will not check for PR merge image. Please publish image manually." && exit 1; - fi - echo "valiator docker image NOT found for [${{ env.TAG }}-${{ env.SHORT_SHA }}]" - git fetch origin pull/${{ steps.findPr.outputs.pr }}/merge:merge && git checkout merge - MERGE_SHORT_SHA=$(git rev-parse HEAD | head -c 8) - echo "Checking valiator image [merge-$MERGE_SHORT_SHA] is present..." - docker pull 0chaindev/validator:merge-$MERGE_SHORT_SHA || { echo "Docker image for valiator could not be found for either branch commit sha [${{ env.SHORT_SHA }}] or PR merge sha [$MERGE_SHORT_SHA]. Please publish image manually and try again." && exit 1; } - echo "valiator docker image found for [merge-$MERGE_SHORT_SHA]" - echo "VALIDATOR_IMAGE=merge-$MERGE_SHORT_SHA" >> $GITHUB_ENV - fi - - - name: "Set PR Status Pending" - uses: niteoweb/pull_request_status_action@v1.0.0 - if: steps.findPr.outputs.number - with: - pr_number: ${{ steps.findPr.outputs.pr }} - description: "Manual system tests running" - state: "pending" - repository: ${{ github.repository }} - context: "0Chain System Tests" - target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: "Deploy 0Chain" - if: ${{ github.event.inputs.skip_tests == 'FALSE' }} - uses: 0chain/actions/deploy-0chain@master - with: - kube_config: ${{ secrets[format('DEV{0}KC', env.RUNNER_NUMBER)] }} - teardown_condition: "TESTS_PASSED" - miner_image: ${{ github.event.inputs.miner_image }} - sharder_image: ${{ github.event.inputs.sharder_image }} - blobber_image: ${{ env.BLOBBER_IMAGE }} - validator_image: ${{ env.VALIDATOR_IMAGE }} - zbox_image: ${{ github.event.inputs.zbox_image }} - zblock_image: ${{ github.event.inputs.zblock_image }} - zdns_image: staging - explorer_image: latest - zproxy_image: staging - zsearch_image: staging - blobber_stake_image: latest - - - name: "Run System tests" - if: ${{ github.event.inputs.skip_tests == 'FALSE' }} - uses: 0chain/actions/run-system-tests@master - with: - system_tests_branch: ${{ github.event.inputs.system_tests_branch }} - network: ${{ env.NETWORK_URL }} - zbox_cli_branch: ${{ github.event.inputs.zbox_cli_branch }} - zwallet_cli_branch: ${{ github.event.inputs.zwallet_cli_branch }} - smart_contract_owner_wallet_json: ${{ secrets.SMART_CONTRACT_OWNER_WALLET_JSON }} - svc_account_secret: ${{ github.token }} - deploy_report_page: false - archive_results: true - run_flaky_tests: false - retry_failures: true - - - name: "Set PR Status Success" - if: ${{ success() && steps.findPr.outputs.number }} - uses: niteoweb/pull_request_status_action@v1.0.0 - with: - pr_number: ${{ steps.findPr.outputs.pr }} - description: "Manual system tests success" - state: ${{ job.status }} - repository: ${{ github.repository }} - context: "0Chain System Tests" - target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: "Set PR Status Failed" - if: ${{ failure() && steps.findPr.outputs.number }} - uses: niteoweb/pull_request_status_action@v1.0.0 - with: - pr_number: ${{ steps.findPr.outputs.pr }} - description: "Manual system tests failure" - state: ${{ job.status }} - repository: ${{ github.repository }} - context: "0Chain System Tests" - target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - env: - GITHUB_TOKEN: ${{ github.token }} + system-tests: + uses: 0chain/actions/.github/workflows/manual_system_tests.yml@master + with: + system_tests_branch: ${{ github.event.inputs.system_tests_branch }} + miner_branch: ${{ github.event.inputs.miner_branch }} + sharder_branch: ${{ github.event.inputs.sharder_branch }} + zbox_cli_branch: ${{ github.event.inputs.zbox_cli_branch }} + zwallet_cli_branch: ${{ github.event.inputs.zwallet_cli_branch }} + blobber_branch: ${{ github.ref_name }} + validator_branch: ${{ github.ref_name }} + zbox_branch: ${{ github.event.inputs.zbox_branch }} + zblock_branch: ${{ github.event.inputs.zblock_branch }} + zdns_branch: ${{ github.event.inputs.zdns_branch }} + explorer_branch: ${{ github.event.inputs.explorer_branch }} + zproxy_branch: staging + zsearch_branch: staging + skip_tests: ${{ github.event.inputs.skip_tests }} + secrets: + SVC_ACCOUNT_SECRET: ${{ secrets.SVC_ACCOUNT_SECRET }} + DEV1KC: ${{ secrets.DEV1KC }} + DEV2KC: ${{ secrets.DEV2KC }} + DEV3KC: ${{ secrets.DEV3KC }} + DEV4KC: ${{ secrets.DEV4KC }} + DEV5KC: ${{ secrets.DEV5KC }} + SMART_CONTRACT_OWNER_WALLET_JSON: ${{ secrets.SMART_CONTRACT_OWNER_WALLET_JSON }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 24b54c7d7..f2d23dae1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,5 +1,9 @@ name: Tests +concurrency: + group: "tests-${{ github.ref }}" + cancel-in-progress: true + on: push: branches: @@ -12,7 +16,7 @@ on: jobs: unit_tests: name: "Unit Tests" - runs-on: [self-hosted, load-test] + runs-on: [self-hosted, build] steps: - name: Setup go 1.17 uses: actions/setup-go@v2 @@ -68,7 +72,7 @@ jobs: golangci: name: lint - runs-on: [self-hosted, tests-suite] + runs-on: [self-hosted, build] steps: - name: Setup go 1.17 uses: actions/setup-go@v2 @@ -83,7 +87,7 @@ jobs: args: --timeout=2m0s buf-build: - runs-on: [self-hosted, tests-suite] + runs-on: [self-hosted, build] steps: - name: Check out code uses: actions/checkout@master diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 5af67e46d..e61b0a50c 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,7 @@ Tasks to complete before merging PR: - [ ] Ensure system tests are passing. If not [Run them manually](https://github.com/0chain/blobber/actions/workflows/system_tests.yml) to check for any regressions :clipboard: -- [ ] Do any new system tests need added to test this change? do any existing system tests need updated? If so create a branch at [0chain/system_test](https://github.com/0chain/system_test) \ No newline at end of file +- [ ] Do any new system tests need added to test this change? do any existing system tests need updated? If so create a PR at [0chain/system_test](https://github.com/0chain/system_test) +- [ ] Merge your system tests PR to master BEFORE merging this PR + +Associated PRs (Link as appropriate): +... \ No newline at end of file From 2427b6fd50ffcffc90fd073bb59d8d415d331853 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 30 Dec 2021 23:54:22 +0800 Subject: [PATCH 080/161] fix(gosdk): upgraded gosdk with 20211230004341-97405e30f6ea (#481) * fix(gosdk): upgraded gosdk with 20211230004341-97405e30f6ea * fix(devops): updated pull request template * fix(devops): fixed runs-on * fix(gomod):go mod tidy * fix(test):fixed rename_test * fix(devops): improved tests.yml * fix(devops): updated runs-on for docker build * fix(devops): improved tests.yml * fix(devops): improved tag for pr * fix(devops): improved tag for pr * fix(devops): improved docker image tag for pr * fix(devops): improved docker image tag for pr * fix(devops): improved docker image tag for pr * fix(devops): fixed invalid syntax on tests.yml * fix(gomod): upgrade gosdk * fix(devops): fixed job names for check * fix(devops):skip-go-installation in golangcli * fix(devops):removed needs from intergation-tests * fix(devops):added cache for go/pkg * fix(devops):fixed syntax on yml * fix(devops):fixed syntax on yml * fix(devops):improved tests.yml * fix(devops):updated runs-on * fix(devops): removed cache actions for go/pkg. download it from github is not faster than goproxy * fix(devops):run system_test with fix/docker_image_tag_for_pr * fix(devops): improved get tag script * fix(devops): simplied job names for dockerize * fix(devops): reverted system_tests to master * fix(devops): improved dockerize, pr template and upgrade gosdk with 3d69e6515423408ebbf99563a73d1996f73ea78c: --- .github/workflows/benchmark.yml | 24 +---- .../build-&-publish-docker-image.yml | 64 ++++++++++--- .github/workflows/tests.yml | 93 ++++++++++--------- PULL_REQUEST_TEMPLATE.md | 6 +- .../allocation/renamefilechange_test.go | 8 +- go.mod | 6 +- go.sum | 61 ++++++++---- 7 files changed, 159 insertions(+), 103 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ef0100349..0dd3c1061 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,27 +22,5 @@ jobs: echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - # - name: Clone gosdk - # run: | - # [ -d $GOSDK ] && rm -rf $GOSDK - # git clone https://github.com/0chain/gosdk.git $GOSDK - # cd $GOSDK - # git checkout $BRANCH - - name: Run benchmark - run: CGO_ENABLED=1 go test -tags bn256 -benchmem -run="BenchmarkUploadFile*" -bench="BenchmarkUploadFile*" ./... | tee benchmark.txt - - # Run `github-action-benchmark` action - # - name: Push benchmark report - # uses: cnlangzi/github-action-benchmark@v1 - # with: - # name: 0chain/gosdk Benchmark - # tool: 'go' - # benchmark-data-dir-path: ${{ env.TAG }}/bench - # output-file-path: benchmark.txt - # # Personal access token to deploy GitHub Pages branch - # github-token: ${{ secrets.GOSDK }} - # #github-token: ${{ secrets.PERSONAL_GITHUB_TOKEN }} - # # Push and deploy GitHub pages branch automatically - # auto-push: true \ No newline at end of file + run: CGO_ENABLED=1 go test -tags bn256 -benchmem -run="BenchmarkUploadFile*" -bench="BenchmarkUploadFile*" ./... | tee benchmark.txt \ No newline at end of file diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index a68592334..4d3a46931 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -9,7 +9,6 @@ on: branches: - staging tags: - - '*' pull_request: workflow_dispatch: @@ -20,15 +19,34 @@ env: VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} jobs: - build_and_publish_blobber: + blobber: runs-on: [self-hosted, build] steps: - - name: Set GITHUB_ENV + - name: Set docker image tag run: | + if [[ "${{github.ref}}" == refs/pull/* ]]; then + tag=${GITHUB_REF/\/merge/} + echo "TAG=$(echo pr-${tag:10})" >> $GITHUB_ENV + else + echo "TAG=$(echo ${GITHUB_REF#refs/*/} | sed 's/\//-/g')" >> $GITHUB_ENV + fi + echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV - echo "TAG=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF) | sed 's/\//-/g')" >> $GITHUB_ENV echo "SHA=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}')" >> $GITHUB_ENV - + + - name: Setup go 1.17 + uses: actions/setup-go@v2 + with: + go-version: '1.17' # The Go version to download (if necessary) and use. + + # - uses: actions/cache@v2 + # with: + # path: ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + # restore-keys: | + # ${{ runner.os }}-go- + + - name: Clone blobber uses: actions/checkout@v1 @@ -53,15 +71,33 @@ jobs: docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA; } - build_and_publish_validator: + validator: runs-on: [self-hosted, build] steps: - - name: Set GITHUB_ENV + - name: Set docker image tag run: | + if [[ "${{github.ref}}" == refs/pull/* ]]; then + tag=${GITHUB_REF/\/merge/} + echo "TAG=$(echo pr-${tag:10})" >> $GITHUB_ENV + else + echo "TAG=$(echo ${GITHUB_REF#refs/*/} | sed 's/\//-/g')" >> $GITHUB_ENV + fi + echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV - echo "TAG=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF) | sed 's/\//-/g')" >> $GITHUB_ENV echo "SHA=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}')" >> $GITHUB_ENV - + + - name: Setup go 1.17 + uses: actions/setup-go@v2 + with: + go-version: '1.17' # The Go version to download (if necessary) and use. + + # - uses: actions/cache@v2 + # with: + # path: ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + # restore-keys: | + # ${{ runner.os }}-go- + - name: Clone blobber uses: actions/checkout@v1 @@ -88,7 +124,7 @@ jobs: system-tests: if: github.event_name != 'workflow_dispatch' - needs: [build_and_publish_blobber, build_and_publish_validator] + needs: [blobber, validator] runs-on: [ tests-suite ] steps: - name: "Get current PR" @@ -112,9 +148,15 @@ jobs: - name: "Setup" run: | + if [[ "${{github.ref}}" == refs/pull/* ]]; then + tag=${GITHUB_REF/\/merge/} + echo "TAG=$(echo pr-${tag:10})" >> $GITHUB_ENV + else + echo "TAG=$(echo ${GITHUB_REF#refs/*/} | sed 's/\//-/g')" >> $GITHUB_ENV + fi + echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV echo "SHORT_SHA=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}') | head -c 8)" >> $GITHUB_ENV - echo "TAG=$(([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF) | sed 's/\//-/g')" >> $GITHUB_ENV echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2d23dae1..55068ce81 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,19 +10,56 @@ on: - master - staging tags: - - "*" pull_request: jobs: - unit_tests: - name: "Unit Tests" - runs-on: [self-hosted, build] + linter: + name: Buf & Golangci lint + runs-on: [self-hosted,docker-builds] steps: - name: Setup go 1.17 uses: actions/setup-go@v2 with: go-version: '1.17' # The Go version to download (if necessary) and use. + # - uses: actions/cache@v2 + # with: + # path: ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + # restore-keys: | + # ${{ runner.os }}-go- + + - name: Clone blobber + uses: actions/checkout@v2 + + - name: Buf-lint + if: success() + run: make local + + - name: Golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: --timeout=2m0s + skip-go-installation: true + + + unit-tests: + name: Unit Tests + runs-on: [self-hosted,docker-builds] + steps: + - name: Setup go 1.17 + uses: actions/setup-go@v2 + with: + go-version: '1.17' # The Go version to download (if necessary) and use. + + # - uses: actions/cache@v2 + # with: + # path: ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + # restore-keys: | + # ${{ runner.os }}-go- + - name: Clone blobber uses: actions/checkout@v2 @@ -31,18 +68,23 @@ jobs: cd $GITHUB_WORKSPACE/code/go/0chain.net/ CGO_ENABLED=1 go test -tags bn256 -race -coverprofile=coverage.txt -covermode=atomic ./... - integration_test: - name: "Integration Tests" - needs: unit_tests + integration-tests: + name: Integration Tests runs-on: [ubuntu-20.04] timeout-minutes: 40 steps: - - name: Setup go 1.17 uses: actions/setup-go@v2 with: go-version: '1.17' # The Go version to download (if necessary) and use. + # - uses: actions/cache@v2 + # with: + # path: ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + # restore-keys: | + # ${{ runner.os }}-go- + - name: Clone Blobber uses: actions/checkout@v1 @@ -68,37 +110,4 @@ jobs: #sudo make integration-tests go17=$(which go) root=$(pwd) - sudo CGO_ENABLED=1 root=$root integration=1 $go17 test -tags bn256 ./... - - golangci: - name: lint - runs-on: [self-hosted, build] - steps: - - name: Setup go 1.17 - uses: actions/setup-go@v2 - with: - go-version: '1.17' # The Go version to download (if necessary) and use. - - uses: actions/checkout@v2 - - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - with: - version: latest - args: --timeout=2m0s - - buf-build: - runs-on: [self-hosted, build] - steps: - - name: Check out code - uses: actions/checkout@master - with: - fetch-depth: 1 - - - name: setup golang - if: success() - uses: actions/setup-go@v2 - with: - go-version: 1.17.x - - name: run buf commands - if: success() - run: make local \ No newline at end of file + sudo CGO_ENABLED=1 root=$root integration=1 $go17 test -tags bn256 ./... \ No newline at end of file diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index e61b0a50c..7e5cc0d4c 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -4,4 +4,8 @@ Tasks to complete before merging PR: - [ ] Merge your system tests PR to master BEFORE merging this PR Associated PRs (Link as appropriate): -... \ No newline at end of file +- zbox: +- zwalelet: +- gosdk: +- 0chain: +- system_test: \ No newline at end of file diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go index dd0c0205a..d1079863b 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -58,7 +58,7 @@ func setupEncryptionScheme() { func setup(t *testing.T) { // setup wallet - w, err := zcncrypto.NewBLS0ChainScheme().GenerateKeys() + w, err := zcncrypto.NewSignatureScheme("bls0chain").GenerateKeys() if err != nil { t.Fatal(err) } @@ -115,9 +115,9 @@ func TestBlobberCore_RenameFile(t *testing.T) { setup(t) setupEncryptionScheme() - sch := zcncrypto.NewBLS0ChainScheme() - sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" - _, err := sch.GenerateKeys() + sch := zcncrypto.NewSignatureScheme("bls0chain") + mnemonic := "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" + _, err := sch.RecoverKeys(mnemonic) if err != nil { t.Fatal(err) } diff --git a/go.mod b/go.mod index c25e7c0a9..b0cca846b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.3.1-0.20211119021259-7c9c46917132 + github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect @@ -24,8 +24,8 @@ require ( github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.19.1 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - google.golang.org/genproto v0.0.0-20211101144312-62acf1d99145 + golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b + google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 google.golang.org/grpc v1.42.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index 197b14017..8f30a534c 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,10 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.3.1-0.20211119021259-7c9c46917132 h1:7dk7FJY0sE79BRz7aH2o+VafMe3GoB2wIXBCpsJfp8o= -github.com/0chain/gosdk v1.3.1-0.20211119021259-7c9c46917132/go.mod h1:8X8+3afqhRRXJcpVeUKKUFvlQ9138eafactRNUMSTL4= +github.com/0chain/gosdk v1.4.1-0.20211230032203-a4d9e51daee3 h1:Eg1pw+KOii9BB+scWzIHSp4rImA2Jybyz01SgLOZxtg= +github.com/0chain/gosdk v1.4.1-0.20211230032203-a4d9e51daee3/go.mod h1:2SyFWa4WQxpomgJxBJrMR4ydWLaLMcqkITY4Pduihi8= +github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423 h1:VcLuPI0PPQb2hqm6fnnFw1CYXpgoykv4zaIbiuORQmk= +github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423/go.mod h1:2SyFWa4WQxpomgJxBJrMR4ydWLaLMcqkITY4Pduihi8= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -127,10 +129,12 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= @@ -212,8 +216,10 @@ github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7z github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY= github.com/didip/tollbooth/v6 v6.1.1 h1:Nt7PvWLa9Y94OrykXsFNBinVRQIu8xdy4avpl99Dc1M= github.com/didip/tollbooth/v6 v6.1.1/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -234,8 +240,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.10.11 h1:KKIcwpmur9iTaVbR2dxlHu+peHVhU+/KX//NWvT1n9U= -github.com/ethereum/go-ethereum v1.10.11/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= +github.com/ethereum/go-ethereum v1.10.4/go.mod h1:nEE0TP5MtxGzOMd7egIrbPJMQBnhVU3ELNxhBglIzhg= +github.com/ethereum/go-ethereum v1.10.13 h1:DEYFP9zk+Gruf3ae1JOJVhNmxK28ee+sMELPLgYTXpA= +github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -293,6 +300,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -359,6 +367,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -435,8 +444,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 h1:rgxjzoDmDXw5q8HONgyHhBas4to0/XWRo/gPpJhsUNQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0/go.mod h1:qrJPVzv9YlhsrxJc3P/Q85nr0w1lIRikTl4JlhdDH5w= -github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= -github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -482,6 +491,7 @@ github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -598,6 +608,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -661,6 +672,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -672,6 +684,7 @@ github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HN github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -682,8 +695,9 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= @@ -691,6 +705,7 @@ github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4f github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miguelmota/go-ethereum-hdwallet v0.1.1/go.mod h1:f9m9uXokAHA6WNoYOPjj4AqjJS5pquQRiYYj/XSyPYc= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -809,8 +824,11 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= -github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= +github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -893,6 +911,7 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -1004,8 +1023,9 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1051,6 +1071,7 @@ golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1097,8 +1118,9 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU= -golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI= +golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1130,6 +1152,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1212,8 +1235,8 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211101204403-39c9dd37992c h1:rnNohYBMnXA07uGnZ9CSWNhIu4Gob4FqWS43lLqZ2sU= -golang.org/x/sys v0.0.0-20211101204403-39c9dd37992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1233,8 +1256,8 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1409,8 +1432,8 @@ google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwy google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20211101144312-62acf1d99145 h1:vum3nDKdleYb+aePXKFEDT2+ghuH00EgYp9B7Q7EZZE= -google.golang.org/genproto v0.0.0-20211101144312-62acf1d99145/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 h1:b9mVrqYfq3P4bCdaLg1qtBnPzUYgglsIdjZkL/fQVOE= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1442,7 +1465,6 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= @@ -1475,8 +1497,9 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.64.0 h1:Mj2zXEXcNb5joEiSA0zc3HZpTst/iyjNiR4CN8tDzOg= +gopkg.in/ini.v1 v1.64.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= From f2e2c36e17c2d2d7c301d0c1692ef67f5433b66c Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 31 Dec 2021 08:12:19 +0800 Subject: [PATCH 081/161] fix(gomod): upgraded deps (#489) * fix(gomod): upgraded deps * fix(grpc):fixed deprecated warning on grpc.WithInsecure --- .../blobbercore/handler/grpc_handler_test.go | 3 +- .../handler/helper_integration_test.go | 3 +- go.mod | 23 ++- go.sum | 155 ++++++++++++++---- 4 files changed, 142 insertions(+), 42 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go index e2e879e2b..128b82296 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" "google.golang.org/grpc/test/bufconn" "gorm.io/gorm" @@ -47,7 +48,7 @@ func makeTestClient() (blobbergrpc.BlobberServiceClient, *grpc.ClientConn, error return lis.Dial() } ) - conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go index ee438f396..0b92ca7cb 100644 --- a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go @@ -13,6 +13,7 @@ import ( blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "gorm.io/driver/postgres" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -56,7 +57,7 @@ func setupHandlerIntegrationTests(t *testing.T) (blobbergrpc.BlobberServiceClien var conn *grpc.ClientConn var err error for i := 0; i < RetryAttempts; i++ { - conn, err = grpc.Dial(BlobberTestAddr, grpc.WithInsecure()) + conn, err = grpc.Dial(BlobberTestAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Println(err) <-time.After(time.Second * RetryTimeout) diff --git a/go.mod b/go.mod index b0cca846b..f9e3b40fa 100644 --- a/go.mod +++ b/go.mod @@ -11,29 +11,34 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.15.0 + github.com/jackc/pgx/v4 v4.14.1 // indirect + github.com/jinzhu/now v1.1.4 // indirect github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible - github.com/mitchellh/mapstructure v1.4.2 + github.com/mitchellh/mapstructure v1.4.3 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce github.com/rs/cors v1.8.0 // indirect github.com/selvatico/go-mocket v1.0.7 - github.com/spf13/viper v1.9.0 + github.com/spf13/afero v1.7.1 // indirect + github.com/spf13/viper v1.10.1 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.19.1 - golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b - google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 - google.golang.org/grpc v1.42.0 - google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb + google.golang.org/grpc v1.43.0 + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 google.golang.org/protobuf v1.27.1 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d - gorm.io/driver/postgres v1.2.2 - gorm.io/gorm v1.22.3 + gorm.io/driver/postgres v1.2.3 + gorm.io/gorm v1.22.4 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 8f30a534c..ef8e632d7 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSR cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -17,6 +18,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -25,6 +27,9 @@ cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSU cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -35,6 +40,7 @@ cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -44,12 +50,11 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.4.1-0.20211230032203-a4d9e51daee3 h1:Eg1pw+KOii9BB+scWzIHSp4rImA2Jybyz01SgLOZxtg= -github.com/0chain/gosdk v1.4.1-0.20211230032203-a4d9e51daee3/go.mod h1:2SyFWa4WQxpomgJxBJrMR4ydWLaLMcqkITY4Pduihi8= github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423 h1:VcLuPI0PPQb2hqm6fnnFw1CYXpgoykv4zaIbiuORQmk= github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423/go.mod h1:2SyFWa4WQxpomgJxBJrMR4ydWLaLMcqkITY4Pduihi8= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= @@ -71,6 +76,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= @@ -105,6 +111,7 @@ github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= @@ -151,6 +158,7 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -160,6 +168,8 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= @@ -170,7 +180,9 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -239,12 +251,15 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/ethereum/go-ethereum v1.10.4/go.mod h1:nEE0TP5MtxGzOMd7egIrbPJMQBnhVU3ELNxhBglIzhg= github.com/ethereum/go-ethereum v1.10.13 h1:DEYFP9zk+Gruf3ae1JOJVhNmxK28ee+sMELPLgYTXpA= github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -261,7 +276,6 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -405,6 +419,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -420,6 +435,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -442,25 +459,30 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 h1:rgxjzoDmDXw5q8HONgyHhBas4to0/XWRo/gPpJhsUNQ= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0/go.mod h1:qrJPVzv9YlhsrxJc3P/Q85nr0w1lIRikTl4JlhdDH5w= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 h1:I/pwhnUln5wbMnTyRbzswA0/JxpK8sZj0aUfI3TV1So= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2/go.mod h1:lsuH8kb4GlMdSlI4alNIBBSAt5CHJtg3i+0WuN9J5YM= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -472,6 +494,7 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -479,10 +502,13 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 h1:TV+x87PxW2BcEzpX6yWK0mBEGc9NFppsihFszD6QmyA= github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -495,6 +521,7 @@ github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1 github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= @@ -527,8 +554,8 @@ github.com/jackc/pgconn v1.6.1/go.mod h1:g8mKMqmSUO6AzAvha7vy07g1rbGOlc7iF0nU0ei github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= -github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8= +github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -546,8 +573,9 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1: github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= @@ -559,8 +587,9 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkAL github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= github.com/jackc/pgtype v1.4.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= -github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.9.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.9.1 h1:MJc2s0MFS8C3ok1wQTdQxWuXQcB6+HwAm5x1CzW7mf0= +github.com/jackc/pgtype v1.9.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -569,13 +598,15 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.7.1/go.mod h1:nu42q3aPjuC1M0Nak4bnoprKlXPINqopEKqbq5AZSC4= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= -github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/pgx/v4 v4.14.0/go.mod h1:jT3ibf/A0ZVCp89rtCIN0zCJxcE74ypROmHEZYsG/j8= +github.com/jackc/pgx/v4 v4.14.1 h1:71oo1KAGI6mXhLiTMn6iDFcp3e7+zon/capWjl2OEFU= +github.com/jackc/pgx/v4 v4.14.1/go.mod h1:RgDuE4Z34o7XE92RpLsvFiOEfrAUT0Xt2KxvX73W06M= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= @@ -584,8 +615,10 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -597,8 +630,9 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -663,6 +697,7 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= @@ -678,8 +713,10 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -691,8 +728,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -705,6 +743,7 @@ github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4f github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miguelmota/go-ethereum-hdwallet v0.1.1/go.mod h1:f9m9uXokAHA6WNoYOPjj4AqjJS5pquQRiYYj/XSyPYc= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= @@ -719,16 +758,18 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= @@ -770,6 +811,7 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -789,6 +831,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -798,6 +841,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -810,6 +854,7 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -844,6 +889,7 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= +github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -874,8 +920,10 @@ github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.7.1 h1:F37zV8E8RLstLpZ0RUGK2NGg1X57y6/B0Eg6S8oqdoA= +github.com/spf13/afero v1.7.1/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -889,8 +937,9 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= +github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= +github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -922,6 +971,7 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= @@ -958,8 +1008,11 @@ go.dedis.ch/protobuf v1.0.11/go.mod h1:97QR256dnkimeNdfmURz0wAMNVbd1VmLXhG1CrTYr go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1020,12 +1073,15 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1066,6 +1122,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1111,16 +1168,20 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1136,6 +1197,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1215,6 +1278,8 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1224,6 +1289,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1235,8 +1301,13 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1319,6 +1390,7 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1365,7 +1437,12 @@ google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtuk google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1413,8 +1490,10 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1431,9 +1510,17 @@ google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKr google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 h1:b9mVrqYfq3P4bCdaLg1qtBnPzUYgglsIdjZkL/fQVOE= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb h1:ZrsicilzPCS/Xr8qtBZZLpy4P9TYXAfl49ctG1/5tgw= +google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1465,10 +1552,13 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= +google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1498,8 +1588,9 @@ gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8 gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.64.0 h1:Mj2zXEXcNb5joEiSA0zc3HZpTst/iyjNiR4CN8tDzOg= gopkg.in/ini.v1 v1.64.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -1530,8 +1621,8 @@ gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d/go.mod h1:n2DTgk9at7cr/CWOT gorm.io/driver/mysql v0.3.1 h1:yvUT7Q0I3B9EHJ67NSp6cHbVwcdDHhVUsDAUiFFxRk0= gorm.io/driver/mysql v0.3.1/go.mod h1:A7H1JD9dKdcjeUTpTuWKEC+E1a74qzW7/zaXqKaTbfM= gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs= -gorm.io/driver/postgres v1.2.2 h1:Ka9W6feOU+rPM9m007eYLMD4QoZuYGBnQ3Jp0faGSwg= -gorm.io/driver/postgres v1.2.2/go.mod h1:Ik3tK+a3FMp8ORZl29v4b3M0RsgXsaeMXh9s9eVMXco= +gorm.io/driver/postgres v1.2.3 h1:f4t0TmNMy9gh3TU2PX+EppoA6YsgFnyq8Ojtddb42To= +gorm.io/driver/postgres v1.2.3/go.mod h1:pJV6RgYQPG47aM1f0QeOzFH9HxQc8JcmAgjRCgS0wjs= gorm.io/driver/sqlite v1.0.8 h1:omllgSb7/eh9D6lGvLZOdU1ZElxdXuO3dn3Rk+dQxUE= gorm.io/driver/sqlite v1.0.8/go.mod h1:xkm8/CEmA3yc4zRd0pdCqm43BjO8Hm6avfTpxWb/7c4= gorm.io/driver/sqlserver v0.2.5 h1:o/MXpn9/BB68RXEEQzfhsSL382yEqUtdCiGIuCspmkY= @@ -1539,9 +1630,9 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.3 h1:/JS6z+GStEQvJNW3t1FTwJwG/gZ+A7crFdRqtvG5ehA= gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.4 h1:8aPcyEJhY0MAt8aY6Dc524Pn+pO29K+ydu+e/cXSpQM= +gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1560,4 +1651,6 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 30916c05fd542598366569087b38a4e9b6ec97db Mon Sep 17 00:00:00 2001 From: prince Date: Fri, 31 Dec 2021 08:50:28 +0530 Subject: [PATCH 082/161] handling for delete root (#453) * Create dir logic added if given path is not available. Signed-off-by: Pradip Parmar * copy_ok testcase query mock change. Signed-off-by: Pradip Parmar * removed createdir methor from obejct operation and added mkdir in ref. Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * testcase fix Signed-off-by: Pradip Parmar * allocation id issue resolved Signed-off-by: Pradip Parmar * handling for delete root Signed-off-by: Pradip Parmar * fix(delete):#451 improved delete logic on reference_objects * query update Signed-off-by: Pradip Parmar * delete refactor and issue fix. Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Lz Co-authored-by: Ryan --- .../allocation/deletefilechange.go | 68 +-------- .../blobbercore/reference/object.go | 130 ++++++++++++++++++ .../0chain.net/blobbercore/reference/ref.go | 6 +- .../0chain.net/blobbercore/reference/vars.go | 5 + 4 files changed, 142 insertions(+), 67 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/reference/object.go create mode 100644 code/go/0chain.net/blobbercore/reference/vars.go diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index 114f7f0c7..7844e768b 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -3,12 +3,10 @@ package allocation import ( "context" "encoding/json" - "path/filepath" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" @@ -25,79 +23,19 @@ type DeleteFileChange struct { } func (nf *DeleteFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { - affectedRef, err := reference.GetObjectTree(ctx, nf.AllocationID, nf.Path) - if err != nil { - return nil, err - } - path, _ := filepath.Split(nf.Path) - path = filepath.Clean(path) - tSubDirs := reference.GetSubDirsFromPath(path) - rootRef, err := reference.GetReferencePath(ctx, nf.AllocationID, nf.Path) + rootRef, contentHash, err := reference.DeleteObject(ctx, nf.AllocationID, nf.Path) if err != nil { return nil, err } - dirRef := rootRef - treelevel := 0 - for treelevel < len(tSubDirs) { - found := false - for _, child := range dirRef.Children { - if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) { - if child.Name == tSubDirs[treelevel] { - dirRef = child - found = true - break - } - } - } - if found { - treelevel++ - } else { - return nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") - } - } - idx := -1 - for i, child := range dirRef.Children { - if child.Hash == nf.Hash && child.Hash == affectedRef.Hash { - idx = i - nf.ContentHash = make(map[string]bool) - if err := reference.DeleteReference(ctx, child.ID, child.PathHash); err != nil { - Logger.Error("DeleteReference", zap.Int64("ref_id", child.ID), zap.Error(err)) - } - if child.Type == reference.FILE { - nf.ContentHash[child.ThumbnailHash] = true - nf.ContentHash[child.ContentHash] = true - } else { - nf.processChildren(ctx, affectedRef) - } - break - } - } - if idx < 0 { - return nil, common.NewError("file_not_found", "Object to delete not found in blobber") - } - - dirRef.RemoveChild(idx) if _, err := rootRef.CalculateHash(ctx, true); err != nil { return nil, err } - return nil, nil -} + nf.ContentHash = contentHash -func (nf *DeleteFileChange) processChildren(ctx context.Context, curRef *reference.Ref) { - for _, childRef := range curRef.Children { - if err := reference.DeleteReference(ctx, childRef.ID, childRef.PathHash); err != nil { - Logger.Error("DeleteReference", zap.Int64("ref_id", childRef.ID), zap.Error(err)) - } - if childRef.Type == reference.FILE { - nf.ContentHash[childRef.ThumbnailHash] = true - nf.ContentHash[childRef.ContentHash] = true - } else if childRef.Type == reference.DIRECTORY { - nf.processChildren(ctx, childRef) - } - } + return nil, nil } func (nf *DeleteFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go new file mode 100644 index 000000000..2cae89491 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -0,0 +1,130 @@ +package reference + +import ( + "context" + "path/filepath" + "strings" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// LoadObjectTree +func LoadObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) { + db := datastore.GetStore(). + GetTransaction(ctx) + + path = filepath.Join("/", path) + db = db.Where("allocation_id = ? and deleted_at IS NULL and path LIKE ? ", allocationID, (path + "%")) + + db = db.Order("level desc, lookup_hash") + + obejctTreeNodes := make(map[string][]*Ref) + + var objects []*Ref + // it is better to load them in batched if there are a lot of objects in db + err := db.FindInBatches(&objects, 100, func(tx *gorm.DB, batch int) error { + // batch processing found records + for _, object := range objects { + + obejctTreeNodes[object.ParentPath] = append(obejctTreeNodes[object.ParentPath], object) + + for _, child := range obejctTreeNodes[object.Path] { + object.AddChild(child) + } + + } + + return nil + }).Error + + if err != nil { + return nil, common.NewError("bad_db_operation", err.Error()) + } + + // create empty dir if root is missing + if len(obejctTreeNodes) == 0 { + return &Ref{Type: DIRECTORY, Path: "/", Name: "/", ParentPath: "", PathLevel: 1}, nil + } + + rootNodes, ok := obejctTreeNodes[""] + + if ok { + if len(rootNodes) == 1 { + return rootNodes[0], nil + } + + return nil, common.NewError("invalid_ref_tree", "/ is missing or invalid") + + } + + return nil, common.NewError("invalid_ref_tree", "/ is missing") + +} + +// DeleteObject delete object from tree, and return tree root and deleted content hash list +func DeleteObject(ctx context.Context, allocationID string, path string) (*Ref, map[string]bool, error) { + + rootRef, err := LoadObjectTree(ctx, allocationID, "/") + if err != nil { + return nil, nil, err + } + + db := datastore.GetStore(). + GetTransaction(ctx) + + var deletedObjects []*Ref + txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "type"}}}) + + path = filepath.Join("/", path) + txDelete = txDelete.Where("allocation_id = ? and deleted_at IS NULL and (path LIKE ? or path = ?) and path != ? ", allocationID, (path + "%"), path, "/") + + err = txDelete.Delete(&deletedObjects).Error + if err != nil { + return nil, nil, common.NewError("bad_db_operation", err.Error()) + } + + deletedFiles := make(map[string]bool) + for _, it := range deletedObjects { + if it.Type == FILE { + deletedFiles[it.ContentHash] = true + } + } + + // remove deleted object from tree + if path == "/" { + rootRef.Children = nil + return rootRef, deletedFiles, nil + } + + path = strings.TrimSuffix(path, "/") + tSubDirs := GetSubDirsFromPath(path) + + dirRef := rootRef + treelevel := 0 + for treelevel < len(tSubDirs)-1 { + found := false + for _, child := range dirRef.Children { + if child.Name == tSubDirs[treelevel] && child.Type == DIRECTORY { + dirRef = child + found = true + break + } + } + if !found { + return nil, nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") + } + treelevel++ + } + + for i, child := range dirRef.Children { + if child.Path == path { + dirRef.RemoveChild(i) + return rootRef, deletedFiles, nil + } + } + return nil, nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") + +} diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 013a88cf9..069b73571 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "math" "path/filepath" "reflect" @@ -131,7 +132,7 @@ type PaginatedRef struct { //Gorm smart select fields. } func (Ref) TableName() string { - return "reference_objects" + return TableNameReferenceObjects } // GetReferenceLookup hash(allocationID + ":" + path) @@ -323,6 +324,7 @@ func (fr *Ref) GetFileHashData() string { func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool) (string, error) { fr.Hash = encryption.Hash(fr.GetFileHashData()) + fmt.Println("file hash", fr.Path, fr.Hash) fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize))) fr.PathHash = GetReferenceLookup(fr.AllocationID, fr.Path) fr.PathLevel = len(GetSubDirsFromPath(fr.Path)) + 1 @@ -360,7 +362,7 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (string, erro } r.Hash = encryption.Hash(strings.Join(childHashes, ":")) - + fmt.Println("ref hash", r.Path, r.Hash) r.NumBlocks = refNumBlocks r.Size = size r.PathHash = encryption.Hash(strings.Join(childPathHashes, ":")) diff --git a/code/go/0chain.net/blobbercore/reference/vars.go b/code/go/0chain.net/blobbercore/reference/vars.go new file mode 100644 index 000000000..13fd7475e --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/vars.go @@ -0,0 +1,5 @@ +package reference + +const ( + TableNameReferenceObjects = "reference_objects" +) From 7994cb43a0b9356f439e2698228cfd4fc69cc29d Mon Sep 17 00:00:00 2001 From: prince Date: Fri, 31 Dec 2021 13:57:22 +0530 Subject: [PATCH 083/161] Bug/issue 401 copy file when no dir (#490) * Create dir logic added if given path is not available. Signed-off-by: Pradip Parmar * copy_ok testcase query mock change. Signed-off-by: Pradip Parmar * removed createdir methor from obejct operation and added mkdir in ref. Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * removed unwanted changes Signed-off-by: Pradip Parmar * testcase fix Signed-off-by: Pradip Parmar * allocation id issue resolved Signed-off-by: Pradip Parmar * fix(delete):#451 improved delete logic on reference_objects * handling for revert on fail for copy command. Signed-off-by: Pradip Parmar * testcases updated for copy_ok. Signed-off-by: Pradip Parmar * trying system test with copy-zbox branch. Signed-off-by: Pradip Parmar * Revert "fix(delete):#451 improved delete logic on reference_objects" This reverts commit 29c5f5227b882e76416500305f8f01977ec9a0a4. * handling for copy to root Signed-off-by: Pradip Parmar * updated testcase branch to validate testcase changes. Signed-off-by: Pradip Parmar * removed testing branch from system-test Signed-off-by: Pradip Parmar * system testcase branch name change Signed-off-by: Pradip Parmar * wallet id reverted Signed-off-by: Pradip Parmar * copy in dir with child fixed Signed-off-by: Pradip Parmar * handling for copy in dir with child. Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Lz --- .../blobbercore/allocation/copyfilechange.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 8ba79f13a..9558dd834 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -45,6 +45,11 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") } + destRef, err = reference.GetRefWithSortedChildren(ctx, rf.AllocationID, rf.DestPath) + if err != nil || destRef.Type != reference.DIRECTORY { + return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") + } + path, _ := filepath.Split(rf.DestPath) path = filepath.Clean(path) tSubDirs := reference.GetSubDirsFromPath(path) @@ -85,8 +90,9 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC return nil, common.NewError("file_not_found", "Destination Object to copy to not found in blobber") } - foundRef := dirRef.Children[childIndex] - rf.processCopyRefs(ctx, affectedRef, foundRef, allocationRoot) + dirRef.RemoveChild(childIndex) + rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot) + dirRef.AddChild(destRef) _, err = rootRef.CalculateHash(ctx, true) From 334f1c56c9f5244139bb98225596e976dd171426 Mon Sep 17 00:00:00 2001 From: prince Date: Sun, 2 Jan 2022 13:51:20 +0530 Subject: [PATCH 084/161] collaborators added in list api (#452) * collaborators added in list api Signed-off-by: Pradip Parmar * show collaborators to owner only. Signed-off-by: Pradip Parmar * removed changes to check system-tests. Signed-off-by: Pradip Parmar * rollback Signed-off-by: Pradip Parmar * nil handling in collaborator list Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Ryan Co-authored-by: Lz --- .../blobbercore/handler/storage_handler.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index e821599b4..1a622ba9d 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -480,6 +480,21 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* if clientID != allocationObj.OwnerID { delete(result.Entities[idx], "path") } + + if child.Type == reference.DIRECTORY || clientID != allocationObj.OwnerID { + continue + } + // getting collaborators for sub dirs + collaborators, err := reference.GetCollaborators(ctx, child.ID) + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + Logger.Error("Failed to get collaborators from refID", zap.Error(err), zap.Any("ref_id", dirref.ID)) + return nil, err + } + if result.Meta["collaborators"] == nil { + result.Meta["collaborators"] = []reference.Collaborator{} + } + result.Meta["collaborators"] = append(result.Meta["collaborators"].([]reference.Collaborator), collaborators...) + } return &result, nil From 2490ac13176a08691664bb4ba257676b6bb923db Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Fri, 14 Jan 2022 20:18:37 +0545 Subject: [PATCH 085/161] Get Refs with Authticket (#495) * Remove redundant check * Add latest read marker in response * Add authticket access to GetRefs function Add latest marker in response * Remove read marker as it cannot be used properly * Fix json missing tag name * Allow other client to access with right credential Check if shareinfo is revoked * Fix conditon checking --- .../handler/object_operation_handler.go | 5 - .../blobbercore/handler/storage_handler.go | 104 ++++++++++++++++-- .../0chain.net/blobbercore/reference/ref.go | 2 +- 3 files changed, 97 insertions(+), 14 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index ad7503165..8a19f0aa8 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -309,11 +309,6 @@ func (fsh *StorageHandler) DownloadFile( payerID = clientID } - if json.Unmarshal([]byte(authTokenString), &readmarker.AuthTicket{}) != nil { - return nil, common.NewErrorf("download_file", - "error parsing the auth ticket for download: %v", err) - } - // we only check content hash if its authticket is referring to a file if authToken.RefType == zfileref.FILE && authToken.ActualFileHash != fileref.ActualFileHash { return nil, errors.New("content hash does not match the requested file content hash") diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 1a622ba9d..5bae8d1a0 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" + "regexp" "strconv" "strings" @@ -56,6 +58,16 @@ func (fsh *StorageHandler) verifyAllocation(ctx context.Context, tx string, return } +func (fsh *StorageHandler) convertGormError(err error) error { + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return common.NewError("invalid_path", "path does not exist") + } + return common.NewError("db_error", err.Error()) + } + return err +} + func (fsh *StorageHandler) verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string) (bool, error) { if len(authTokenString) == 0 { return false, common.NewError("invalid_parameters", "Auth ticket required if data read by anyone other than owner.") @@ -730,25 +742,100 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } + clientID := ctx.Value(constants.ContextKeyClient).(string) + if len(clientID) == 0 { + return nil, common.NewError("invalid_operation", "Client id is required") + } + + publicKey, _ := ctx.Value(constants.ContextKeyClientKey).(string) + if publicKey == "" { + if clientID == allocationObj.OwnerID { + publicKey = allocationObj.OwnerPublicKey + } else { + return nil, common.NewError("empty_public_key", "public key is required") + } + } + clientSign, _ := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string) - valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) + + valid, err := verifySignatureFromRequest(allocationTx, clientSign, publicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } allocationID := allocationObj.ID - clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } + path := r.FormValue("path") - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") + authTokenStr := r.FormValue("auth_token") + pathHash := r.FormValue("path_hash") + + if path == "" && authTokenStr == "" && pathHash == "" { + return nil, common.NewError("invalid_parameters", "empty path and authtoken") } + var pathRef *reference.Ref + switch { + case path != "": + pathHash = reference.GetReferenceLookup(allocationID, path) + fallthrough + case pathHash != "": + pathRef, err = reference.GetReferenceFromLookupHash(ctx, allocationID, pathHash) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, common.NewError("invalid_path", "") + } + return nil, err + } + + if clientID == allocationObj.OwnerID || clientID == allocationObj.RepairerID || reference.IsACollaborator(ctx, pathRef.ID, clientID) { + break + } + if authTokenStr == "" { + return nil, common.NewError("unauthorized_request", "client is not authorized for the requested resource") + } + fallthrough + case authTokenStr != "": + authToken := &readmarker.AuthTicket{} + if err := json.Unmarshal([]byte(authTokenStr), authToken); err != nil { + return nil, common.NewError("json_unmarshall_error", fmt.Sprintf("error parsing authticket: %v", authTokenStr)) + } + + shareInfo, err := reference.GetShareInfo(ctx, authToken.ClientID, authToken.FilePathHash) + if err != nil { + return nil, fsh.convertGormError(err) + } + if shareInfo.Revoked { + return nil, common.NewError("share_revoked", "client no longer has permission to requested resource") + } + + if err := authToken.Verify(allocationObj, clientID); err != nil { + return nil, err + } + + if pathRef == nil { + pathRef, err = reference.GetReferenceFromLookupHash(ctx, allocationID, authToken.FilePathHash) + if err != nil { + return nil, fsh.convertGormError(err) + } + } else if pathHash != authToken.FilePathHash { + authTokenRef, err := reference.GetReferenceFromLookupHash(ctx, allocationID, authToken.FilePathHash) + if err != nil { + return nil, fsh.convertGormError(err) + } + matched, _ := regexp.MatchString(fmt.Sprintf("^%v", authTokenRef.Path), pathRef.Path) + if !matched { + return nil, common.NewError("invalid_authticket", "auth ticket is not valid for requested resource") + } + } + default: + return nil, common.NewError("incomplete_request", "path, pathHash or authTicket is required") + + } + + path = pathRef.Path pageLimitStr := r.FormValue("pageLimit") var pageLimit int - if len(pageLimitStr) == 0 { + if pageLimitStr == "" { pageLimit = PageLimit } else { o, err := strconv.Atoi(pageLimitStr) @@ -763,6 +850,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb pageLimit = o } } + offsetPath := r.FormValue("offsetPath") offsetDate := r.FormValue("offsetDate") updatedDate := r.FormValue("updatedDate") diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 069b73571..078153fb1 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -128,7 +128,7 @@ type PaginatedRef struct { //Gorm smart select fields. UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at,omitempty"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at" json:"-"` // soft deletion - ChunkSize int64 `gorm:"column:chunk_size" dirlist:"chunk_size" filelist:"chunk_size"` + ChunkSize int64 `gorm:"column:chunk_size" json:"chunk_size"` } func (Ref) TableName() string { From 178e93f8abfaebb448f457ba6cb90a382fa28050 Mon Sep 17 00:00:00 2001 From: prince Date: Fri, 14 Jan 2022 21:00:22 +0530 Subject: [PATCH 086/161] https flags added in blobber server. (#500) * https flags added in blobber server. Signed-off-by: Pradip Parmar * if both port available we need to listen on both: http and https Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Lz --- code/go/0chain.net/blobber/flags.go | 14 +++++++-- code/go/0chain.net/blobber/http.go | 44 ++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go index 92c50a2cb..aa037f483 100644 --- a/code/go/0chain.net/blobber/flags.go +++ b/code/go/0chain.net/blobber/flags.go @@ -18,6 +18,9 @@ var ( configDir string grpcPort int isIntegrationTest bool + httpsPort int + httpsKeyFile string + httpsCertFile string ) func init() { @@ -29,6 +32,9 @@ func init() { flag.StringVar(&metadataDB, "db_dir", "", "db_dir") flag.StringVar(&logDir, "log_dir", "", "log_dir") flag.IntVar(&httpPort, "port", 0, "port") + flag.IntVar(&httpsPort, "https-port", 0, "https-port") + flag.StringVar(&httpsCertFile, "https-cert-file", "", "https-cert-file") + flag.StringVar(&httpsKeyFile, "https-key-file", "", "https-key-file") flag.StringVar(&hostname, "hostname", "", "hostname") flag.StringVar(&configDir, "config_dir", "./config", "config_dir") @@ -51,10 +57,14 @@ func parseFlags() { panic("Please specify --hostname which is the public hostname") } - if httpPort <= 0 { - panic("Please specify --port which is the port on which requests are accepted") + if httpPort <= 0 && httpsPort <= 0 { + panic("Please specify --port or --https-port which is the port on which requests are accepted") } isIntegrationTest = os.Getenv("integration") == "1" + if httpsPort > 0 && (httpsCertFile == "" || httpsKeyFile == "") { + panic("Please specify --https-cert-file and --https-key-file if you are using --https-port") + } + fmt.Print(" [OK]\n") } diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index 0886dbdd9..4f634e903 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -6,6 +6,7 @@ import ( "net/http" "runtime" "strconv" + "sync" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -25,16 +26,38 @@ func startHttpServer() { mode = "test net" } - logging.Logger.Info("Starting blobber", zap.Int("available_cpus", runtime.NumCPU()), zap.Int("port", httpPort), zap.String("chain_id", config.GetServerChainID()), zap.String("mode", mode)) - - //address := publicIP + ":" + portString - address := ":" + strconv.Itoa(httpPort) - var server *http.Server - common.ConfigRateLimits() r := mux.NewRouter() initHandlers(r) + var wg sync.WaitGroup + + wg.Add(2) + // start http server + go startServer(&wg, r, mode, httpPort, false) + // start https server + go startServer(&wg, r, mode, httpsPort, true) + + logging.Logger.Info("Ready to listen to the requests") + fmt.Println("[11/11] start http server [OK]") + + wg.Wait() + +} + +func startServer(wg *sync.WaitGroup, r *mux.Router, mode string, port int, isTls bool) { + defer wg.Done() + + if port <= 0 { + return + } + + logging.Logger.Info("Starting blobber", zap.Int("available_cpus", runtime.NumCPU()), zap.Int("port", port), zap.Bool("is-tls", isTls), zap.String("chain_id", config.GetServerChainID()), zap.String("mode", mode)) + + //address := publicIP + ":" + portString + address := ":" + strconv.Itoa(port) + var server *http.Server + if config.Development() { // No WriteTimeout setup to enable pprof server = &http.Server{ @@ -56,10 +79,11 @@ func startHttpServer() { common.HandleShutdown(server) handler.HandleShutdown(common.GetRootContext()) - logging.Logger.Info("Ready to listen to the requests") - fmt.Println("[11/11] start http server [OK]") - - log.Fatal(server.ListenAndServe()) + if isTls { + log.Fatal(server.ListenAndServeTLS(httpsCertFile, httpsKeyFile)) + } else { + log.Fatal(server.ListenAndServe()) + } } func initHandlers(r *mux.Router) { From 9213cb20af164eeba023c354a44aaac8799944db Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 15 Jan 2022 07:07:17 +0800 Subject: [PATCH 087/161] fix(gomod): upgrade gosdk, gorm and zap (#501) --- go.mod | 7 +++---- go.sum | 15 ++++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index f9e3b40fa..8fae16e3d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423 + github.com/0chain/gosdk v1.4.1-0.20220105220554-257eaff7f1d6 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect @@ -15,7 +15,6 @@ require ( github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.15.0 github.com/jackc/pgx/v4 v4.14.1 // indirect - github.com/jinzhu/now v1.1.4 // indirect github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/mapstructure v1.4.3 @@ -26,7 +25,7 @@ require ( github.com/spf13/viper v1.10.1 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 - go.uber.org/zap v1.19.1 + go.uber.org/zap v1.20.0 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect @@ -38,7 +37,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.2.3 - gorm.io/gorm v1.22.4 + gorm.io/gorm v1.22.5 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index ef8e632d7..038f08c0c 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423 h1:VcLuPI0PPQb2hqm6fnnFw1CYXpgoykv4zaIbiuORQmk= -github.com/0chain/gosdk v1.4.1-0.20211230114941-3d69e6515423/go.mod h1:2SyFWa4WQxpomgJxBJrMR4ydWLaLMcqkITY4Pduihi8= +github.com/0chain/gosdk v1.4.1-0.20220105220554-257eaff7f1d6 h1:O7dKFWSLQyctymZqKLgXYmK/15i1gxxXFsaP1JwHiAQ= +github.com/0chain/gosdk v1.4.1-0.20220105220554-257eaff7f1d6/go.mod h1:FB2xXhQyIM1vwvQ1jC98wNclbDTBwqrG+Z/IQC0LaBs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -616,7 +616,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -1031,8 +1030,9 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -1047,8 +1047,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= +go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1631,8 +1632,8 @@ gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.4 h1:8aPcyEJhY0MAt8aY6Dc524Pn+pO29K+ydu+e/cXSpQM= -gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= +gorm.io/gorm v1.22.5 h1:lYREBgc02Be/5lSCTuysZZDb6ffL2qrat6fg9CFbvXU= +gorm.io/gorm v1.22.5/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 112c4bdaef8bb334fb02462602ed090ecaa60da1 Mon Sep 17 00:00:00 2001 From: prince Date: Mon, 17 Jan 2022 09:24:40 +0530 Subject: [PATCH 088/161] removed physical createdir, updated createdir default size. (#491) * removed physical createdir, updated createdir default size. Signed-off-by: Pradip Parmar * issues/441 duplicate dir creation handling. Signed-off-by: Pradip Parmar * unit test fix Signed-off-by: Pradip Parmar * issue handling Signed-off-by: Pradip Parmar * removed createDir from fileStore. Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Lz Co-authored-by: Kishan-Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> --- .../0chain.net/blobbercore/allocation/newfilechange.go | 10 ++++++---- code/go/0chain.net/blobbercore/filestore/fs_store.go | 4 ---- code/go/0chain.net/blobbercore/filestore/store.go | 1 - .../blobbercore/handler/object_operation_handler.go | 5 ----- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/newfilechange.go b/code/go/0chain.net/blobbercore/allocation/newfilechange.go index 0428f6409..5b48dde8c 100644 --- a/code/go/0chain.net/blobbercore/allocation/newfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/newfilechange.go @@ -94,13 +94,15 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a } } + // adding nil to make childLoaded as true so we can have hash calculated in CalculateHas. + // without has commit fails var newDir = reference.NewDirectoryRef() - newDir.ActualFileSize = 2 + newDir.ActualFileSize = 0 newDir.AllocationID = dirRef.AllocationID newDir.MerkleRoot = nf.MerkleRoot - newDir.Name = dirName - newDir.Size = 2 - newDir.NumBlocks = 2 + newDir.Path = dirName + newDir.Size = 0 + newDir.NumBlocks = 0 newDir.ParentPath = dirRef.Path newDir.WriteMarker = allocationRoot dirRef.AddChild(newDir) diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index a3223eb58..0dc6c2572 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -450,10 +450,6 @@ func (fs *FileFSStore) DeleteFile(allocationID string, contentHash string) error return os.Remove(fileObjectPath) } -func (fs *FileFSStore) CreateDir(dirName string) error { - return createDirs(dirName) -} - func (fs *FileFSStore) DeleteDir(allocationID, dirPath, connectionID string) error { return nil } diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index 305ad2165..dc3a1ad21 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -45,7 +45,6 @@ type FileStore interface { WriteFile(allocationID string, fileData *FileInputData, infile multipart.File, connectionID string) (*FileOutputData, error) DeleteTempFile(allocationID string, fileData *FileInputData, connectionID string) error - CreateDir(dirName string) error DeleteDir(allocationID, dirPath, connectionID string) error GetFileBlock(allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 8a19f0aa8..d64b5c72b 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1051,11 +1051,6 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo connectionObj.AddChange(allocationChange, &formData) - err = filestore.GetFileStore().CreateDir(dirPath) - if err != nil { - return nil, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) - } - err = connectionObj.ApplyChanges(ctx, "/") if err != nil { return nil, err From a8d54da20cf46819eb397e0c65555661cc2d2a9b Mon Sep 17 00:00:00 2001 From: prince Date: Thu, 20 Jan 2022 17:59:10 +0530 Subject: [PATCH 089/161] readme updated for https. (#505) * readme updated for https. Signed-off-by: Pradip Parmar * https docker-compose for example. Signed-off-by: Pradip Parmar * cert path changed. Signed-off-by: Pradip Parmar * port exported. Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar Co-authored-by: Lz --- config/localhost.crt | 17 ++++++ config/localhost.key | 9 +++ docker.aws/README.md | 12 +++- docker.local/https.docker-compose.yml | 88 +++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 config/localhost.crt create mode 100644 config/localhost.key create mode 100644 docker.local/https.docker-compose.yml diff --git a/config/localhost.crt b/config/localhost.crt new file mode 100644 index 000000000..5996c0fcf --- /dev/null +++ b/config/localhost.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICvDCCAkKgAwIBAgIUdBMs5yZ7wQS8ZQFDGaIgURs7TdMwCgYIKoZIzj0EAwIw +gZQxCzAJBgNVBAYTAklOMRAwDgYDVQQIDAdHdWphcmF0MQ4wDAYDVQQHDAVTdXJh +dDEUMBIGA1UECgwLQWxmYUNyZWF0b3IxFDASBgNVBAsMC0RldmVsb3BtZW50MQ8w +DQYDVQQDDAZQcmFkaXAxJjAkBgkqhkiG9w0BCQEWF3ByYWRpcEBhbGZhLWNyZWF0 +b3IuY29tMB4XDTIyMDExNDAyMDUwNVoXDTMyMDExMjAyMDUwNVowgZQxCzAJBgNV +BAYTAklOMRAwDgYDVQQIDAdHdWphcmF0MQ4wDAYDVQQHDAVTdXJhdDEUMBIGA1UE +CgwLQWxmYUNyZWF0b3IxFDASBgNVBAsMC0RldmVsb3BtZW50MQ8wDQYDVQQDDAZQ +cmFkaXAxJjAkBgkqhkiG9w0BCQEWF3ByYWRpcEBhbGZhLWNyZWF0b3IuY29tMHYw +EAYHKoZIzj0CAQYFK4EEACIDYgAE/vPDjHmDzM/bsAQFm8Njq+DwlN9Vj73ZU0Pg +PGHgpJicvUTpl7LyrbmlGPJL8gwDZn3pkpJN906ER6Y8zGqHUATxWf4f6014+zDo +EjroaDaFTl1dIsXX8BCgcXEMKQ3Fo1MwUTAdBgNVHQ4EFgQU047AQXH061UPttjb +SIFmiAryHgIwHwYDVR0jBBgwFoAU047AQXH061UPttjbSIFmiAryHgIwDwYDVR0T +AQH/BAUwAwEB/zAKBggqhkjOPQQDAgNoADBlAjEAypdZMUc/8OYXICh7zONovLpX +d5OrKL7OXgeqTQ4GaLZvCB/mvRAyApONAiuL7EleAjB6uDoTI0JbOM2LQpECx7JE +C0w0e34Usc6/7S+DciPHSeG2oF6P1yP/bQ+QPVSRCbo= +-----END CERTIFICATE----- diff --git a/config/localhost.key b/config/localhost.key new file mode 100644 index 000000000..08c58321f --- /dev/null +++ b/config/localhost.key @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDCyVug6uG+OfbkNxOtdK55M2+G143LeVb2j7z5L7comr9as3Ff10PGv +Dv89Z4e1V8GgBwYFK4EEACKhZANiAAT+88OMeYPMz9uwBAWbw2Or4PCU31WPvdlT +Q+A8YeCkmJy9ROmXsvKtuaUY8kvyDANmfemSkk33ToRHpjzMaodQBPFZ/h/rTXj7 +MOgSOuhoNoVOXV0ixdfwEKBxcQwpDcU= +-----END EC PRIVATE KEY----- diff --git a/docker.aws/README.md b/docker.aws/README.md index 4e68b7c5d..25f8648ee 100644 --- a/docker.aws/README.md +++ b/docker.aws/README.md @@ -50,9 +50,15 @@ block_worker: https://beta.0chain.net/dns ` command: ./bin/blobber --port 505${BLOBBER} --hostname < public ip here > --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data --minio_file keysconfig/minio_config.txt ` - -5. Go to git/blobber directory to build containers using + +5. To listen on tls network add `--https-port`, `--https-cert-file` (certificate file) and `--https-key-file` (key file) + + +` command: ./bin/blobber --port 505${BLOBBER} --https-port 506${BLOBBER} --https-cert-file < certificate file > --https-key-file < key file > --hostname < public ip here > --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data --minio_file keysconfig/minio_config.txt ` + + +6. Go to git/blobber directory to build containers using ``` @@ -62,7 +68,7 @@ $ ./docker.local/bin/build.blobber.sh ``` -6. After building the container for blobber, go to Blobber1 directory (git/blobber/docker.local/blobber1) and run the container using +7. After building the container for blobber, go to Blobber1 directory (git/blobber/docker.local/blobber1) and run the container using diff --git a/docker.local/https.docker-compose.yml b/docker.local/https.docker-compose.yml new file mode 100644 index 000000000..83e5b996b --- /dev/null +++ b/docker.local/https.docker-compose.yml @@ -0,0 +1,88 @@ +version: "3" +services: + postgres: + image: postgres:11 + volumes: + - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data + networks: + default: + ports: + - "543${BLOBBER}:5432" + labels: + zchain: "postgres" + postgres-post: + image: postgres:11 + environment: + POSTGRES_PORT: 5432 + POSTGRES_HOST: postgres + POSTGRES_USER: postgres + volumes: + - ../bin:/blobber/bin + - ../sql:/blobber/sql + labels: + zchain: "postgres-post" + command: bash /blobber/bin/postgres-entrypoint.sh + links: + - postgres:postgres + validator: + image: validator + environment: + - DOCKER= true + depends_on: + - postgres-post + links: + - postgres-post:postgres-post + volumes: + - ../config:/blobber/config + - ./blobber${BLOBBER}/data:/blobber/data + - ./blobber${BLOBBER}/log:/blobber/log + - ./keys_config:/blobber/keysconfig + ports: + - "506${BLOBBER}:506${BLOBBER}" + labels: + zchain: "validator" + command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --log_dir /blobber/log + networks: + default: + testnet0: + ipv4_address: 198.18.0.6${BLOBBER} + + blobber: + image: blobber + depends_on: + - validator + links: + - validator:validator + volumes: + - ../config:/blobber/config + - ./blobber${BLOBBER}/files:/blobber/files + - ./blobber${BLOBBER}/data:/blobber/data + - ./blobber${BLOBBER}/log:/blobber/log + - ./keys_config:/blobber/keysconfig + ports: + - "505${BLOBBER}:505${BLOBBER}" + - "506${BLOBBER}:506${BLOBBER}" + - "703${BLOBBER}:703${BLOBBER}" + labels: + zchain: "blobber" + command: ./bin/blobber --port 505${BLOBBER} --https-port 506${BLOBBER} --https-cert-file config/localhost.crt --https-key-file config/localhost.key --grpc_port 703${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data --minio_file keys_config/minio_config.txt + networks: + default: + testnet0: + ipv4_address: 198.18.0.9${BLOBBER} + +networks: + default: + driver: bridge + testnet0: + external: true + # driver: bridge + # ipam: + # config: + # - subnet: 198.18.0.0/15 + # - gateway: 198.18.0.255 + +volumes: + data: + config: + bin: From 741e9993354ff55124b256a870fca34b21bb2aa3 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 21 Jan 2022 04:17:05 +0800 Subject: [PATCH 090/161] fix(main): register with https on zcn (#510) * fix(main): register with https on zcn * fix(main): register blobber with https on zcn --- code/go/0chain.net/blobber/node.go | 7 ++++++- code/go/0chain.net/core/node/self_node.go | 9 +++++++-- code/go/0chain.net/validator/main.go | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index 2d37fa9d8..e5e939ad3 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -30,7 +30,12 @@ func setupNode() error { logging.Logger.Info("self identity", zap.Any("id", node.Self.ID)) } - node.Self.SetHostURL(hostname, httpPort) + if httpsPort > 0 { + node.Self.SetHostURL("https", hostname, httpsPort) + } else { + node.Self.SetHostURL("http", hostname, httpPort) + } + logging.Logger.Info(" Base URL" + node.Self.GetURLBase()) fmt.Print(" [OK]\n") return nil diff --git a/code/go/0chain.net/core/node/self_node.go b/code/go/0chain.net/core/node/self_node.go index 19afb1369..6c1a0ad51 100644 --- a/code/go/0chain.net/core/node/self_node.go +++ b/code/go/0chain.net/core/node/self_node.go @@ -38,11 +38,16 @@ func (sn *SelfNode) SetKeys(publicKey string, privateKey string) { } /*SetHostURL - setter for Host and Port */ -func (sn *SelfNode) SetHostURL(address string, port int) { +func (sn *SelfNode) SetHostURL(schema, address string, port int) { if address == "" { address = "localhost" } - sn.URL = fmt.Sprintf("http://%v:%v", address, port) + + if schema == "" { + schema = "http" + } + + sn.URL = fmt.Sprintf("%v://%v:%v", schema, address, port) } /*GetURLBase - get the end point base */ diff --git a/code/go/0chain.net/validator/main.go b/code/go/0chain.net/validator/main.go index 7a1b3e0c0..413d647d3 100644 --- a/code/go/0chain.net/validator/main.go +++ b/code/go/0chain.net/validator/main.go @@ -96,7 +96,7 @@ func main() { return } - node.Self.SetHostURL(*hostname, port) + node.Self.SetHostURL("http", *hostname, port) Logger.Info(" Base URL" + node.Self.GetURLBase()) config.SetServerChainID(config.Configuration.ChainID) From 4ecc324036f508cc74aa4e49795a9745c6d0c986 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 21 Jan 2022 10:14:39 +0800 Subject: [PATCH 091/161] fix(gomod):upgraded grpc packages (#512) * fix(gomod):upgraded grpc packages * fix(gomod):upgraded gosdk * fix(docker): removed debugging gcflags for compile optimizations * fix(docker): don't use git on run docker image * fix(docker): use alpine:3.14 instead of golang image for production * fix(devops): added concurrency for benchmark --- .github/workflows/benchmark.yml | 16 +++++++--------- docker.local/Dockerfile | 10 +++------- docker.local/blobber.Dockerfile | 4 ++-- docker.local/validator.Dockerfile | 2 +- go.mod | 6 +++--- go.sum | 15 ++++++++------- 6 files changed, 24 insertions(+), 29 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0dd3c1061..d815d4c30 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,26 +1,24 @@ name: Benchmark tests +concurrency: + group: "benchmark-${{ github.ref }}" + cancel-in-progress: true + on: push: branches: [ master, staging ] jobs: benchmark: - runs-on: [self-hosted, load-test] + runs-on: [self-hosted, conductor-test] steps: - - name: Setup go 1.16 + - name: Setup go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' # The Go version to download (if necessary) and use. + go-version: '1.17' # The Go version to download (if necessary) and use. - name: Clone blobber uses: actions/checkout@v2 - - name: Set GITHUB_ENV - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - echo "GOSDK=$(echo $(dirname $(pwd)))/gosdk" >> $GITHUB_ENV - echo "TAG=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g' )" >> $GITHUB_ENV - - name: Run benchmark run: CGO_ENABLED=1 go test -tags bn256 -benchmem -run="BenchmarkUploadFile*" -bench="BenchmarkUploadFile*" ./... | tee benchmark.txt \ No newline at end of file diff --git a/docker.local/Dockerfile b/docker.local/Dockerfile index 4fdc70a96..b1e4cf0b9 100644 --- a/docker.local/Dockerfile +++ b/docker.local/Dockerfile @@ -19,19 +19,15 @@ WORKDIR $SRC_DIR/code/go/0chain.net/blobber ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT -RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -gcflags "all=-N -l" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" +RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.17.1-alpine3.14 -RUN apk add gmp gmp-dev openssl-dev git +FROM alpine:3.14 +RUN apk add gmp gmp-dev openssl-dev COPY --from=blobber_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ /usr/local/lib/ -RUN git clone --branch v1.4.1 https://github.com/go-delve/delve -WORKDIR ./delve -RUN go install ./cmd/dlv - ENV APP_DIR=/blobber WORKDIR $APP_DIR COPY --from=blobber_build $APP_DIR/code/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index 9e93e1247..db01b09fa 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -19,8 +19,8 @@ ENV GIT_COMMIT=$GIT_COMMIT RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.17.1-alpine3.14 -RUN apk add gmp gmp-dev openssl-dev git +FROM alpine:3.14 +RUN apk add gmp gmp-dev openssl-dev COPY --from=blobber_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ /usr/local/lib/ diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index 62191b7a7..cc3f284c4 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -20,7 +20,7 @@ WORKDIR $SRC_DIR/code/go/0chain.net/validator RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" # Copy the build artifact into a minimal runtime image: -FROM golang:1.17.1-alpine3.14 +FROM alpine:3.14 RUN apk add gmp gmp-dev openssl-dev COPY --from=validator_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ diff --git a/go.mod b/go.mod index 8fae16e3d..3cc92ef50 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.4.1-0.20220105220554-257eaff7f1d6 + github.com/0chain/gosdk v1.4.1-0.20220117131359-7dd554e5a0f5 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.15.0 github.com/jackc/pgx/v4 v4.14.1 // indirect @@ -29,7 +29,7 @@ require ( golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect - google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb + google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 google.golang.org/grpc v1.43.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index 038f08c0c..ecf7ab316 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.4.1-0.20220105220554-257eaff7f1d6 h1:O7dKFWSLQyctymZqKLgXYmK/15i1gxxXFsaP1JwHiAQ= -github.com/0chain/gosdk v1.4.1-0.20220105220554-257eaff7f1d6/go.mod h1:FB2xXhQyIM1vwvQ1jC98wNclbDTBwqrG+Z/IQC0LaBs= +github.com/0chain/gosdk v1.4.1-0.20220117131359-7dd554e5a0f5 h1:39ZQnIjlTe0JBIn8wrLQqMMFhbZX900qdMSveO8EUhc= +github.com/0chain/gosdk v1.4.1-0.20220117131359-7dd554e5a0f5/go.mod h1:FB2xXhQyIM1vwvQ1jC98wNclbDTBwqrG+Z/IQC0LaBs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -402,8 +402,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -459,8 +460,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 h1:I/pwhnUln5wbMnTyRbzswA0/JxpK8sZj0aUfI3TV1So= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2/go.mod h1:lsuH8kb4GlMdSlI4alNIBBSAt5CHJtg3i+0WuN9J5YM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 h1:I8MsauTJQXZ8df8qJvEln0kYNc3bSapuaSsEsnFdEFU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3/go.mod h1:lZdb/YAJUSj9OqrCHs2ihjtoO3+xK3G53wTYXFWRGDo= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -1520,8 +1521,8 @@ google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb h1:ZrsicilzPCS/Xr8qtBZZLpy4P9TYXAfl49ctG1/5tgw= -google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 h1:zzNejm+EgrbLfDZ6lu9Uud2IVvHySPl8vQzf04laR5Q= +google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= From 3e5b77a0cdd8c4702bf0dfa82b0779e586a462d2 Mon Sep 17 00:00:00 2001 From: Lz Date: Sun, 23 Jan 2022 19:33:37 +0800 Subject: [PATCH 092/161] fix(node): added cli argument --hosturl to register url on zcn (#513) * fix(node): added cli argument --hosturl to register url on zcn * fix(node): fixed synatx issue --- code/go/0chain.net/blobber/flags.go | 9 ++++++--- code/go/0chain.net/blobber/node.go | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go index aa037f483..436c23467 100644 --- a/code/go/0chain.net/blobber/flags.go +++ b/code/go/0chain.net/blobber/flags.go @@ -21,6 +21,7 @@ var ( httpsPort int httpsKeyFile string httpsCertFile string + hostUrl string ) func init() { @@ -32,12 +33,14 @@ func init() { flag.StringVar(&metadataDB, "db_dir", "", "db_dir") flag.StringVar(&logDir, "log_dir", "", "log_dir") flag.IntVar(&httpPort, "port", 0, "port") - flag.IntVar(&httpsPort, "https-port", 0, "https-port") - flag.StringVar(&httpsCertFile, "https-cert-file", "", "https-cert-file") - flag.StringVar(&httpsKeyFile, "https-key-file", "", "https-key-file") + flag.IntVar(&httpsPort, "https_port", 0, "https_port") + flag.StringVar(&httpsCertFile, "https_cert_file", "", "https_cert_file") + flag.StringVar(&httpsKeyFile, "https_key_file", "", "https_key_file") flag.StringVar(&hostname, "hostname", "", "hostname") flag.StringVar(&configDir, "config_dir", "./config", "config_dir") + flag.StringVar(&hostUrl, "hosturl", "", "register url on blockchain instead of [schema://hostname+port] if it has value") + flag.IntVar(&grpcPort, "grpc_port", 0, "grpc_port") } diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index e5e939ad3..c2322903c 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -30,10 +30,14 @@ func setupNode() error { logging.Logger.Info("self identity", zap.Any("id", node.Self.ID)) } - if httpsPort > 0 { - node.Self.SetHostURL("https", hostname, httpsPort) + if len(hostUrl) > 0 { + node.Self.URL = hostUrl } else { - node.Self.SetHostURL("http", hostname, httpPort) + if httpsPort > 0 { + node.Self.SetHostURL("https", hostname, httpsPort) + } else { + node.Self.SetHostURL("http", hostname, httpPort) + } } logging.Logger.Info(" Base URL" + node.Self.GetURLBase()) From caf05c9dd9779c68919538c4f9eee9dfcaab4142 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 26 Jan 2022 18:01:30 +0800 Subject: [PATCH 093/161] fix(upload): thumbnail is lost (#516) * fix(upload): thumbnail is not committed correcly on upload/update request * fix(devops): short name for UI pretty on github checks * fix(devops): short name for UI pretty on github checks --- .../build-&-publish-docker-image.yml | 2 +- .github/workflows/tests.yml | 2 +- .../allocation/allocationchange.go | 3 +- .../allocation/file_changer_base.go | 1 + .../blobbercore/handler/file_command_add.go | 4 + .../handler/file_command_insert.go | 135 ------------------ .../handler/file_command_update.go | 5 +- 7 files changed, 12 insertions(+), 140 deletions(-) delete mode 100644 code/go/0chain.net/blobbercore/handler/file_command_insert.go diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 4d3a46931..ae879038c 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -1,4 +1,4 @@ -name: "Build & Publish Docker Image" +name: "Dockerize" concurrency: group: "publish-${{ github.ref }}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 55068ce81..c1ee840d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: jobs: linter: - name: Buf & Golangci lint + name: Lints runs-on: [self-hosted,docker-builds] steps: - name: Setup go 1.17 diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index e7bc51ebc..7ff70c93b 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -119,7 +119,7 @@ func (cc *AllocationChangeCollector) ComputeProperties() { var acp AllocationChangeProcessor switch change.Operation { case constants.FileOperationInsert: - acp = new(NewFileChange) + acp = new(AddFileChanger) case constants.FileOperationUpdate: acp = new(UpdateFileChanger) case constants.FileOperationDelete: @@ -130,7 +130,6 @@ func (cc *AllocationChangeCollector) ComputeProperties() { acp = new(CopyFileChange) case constants.FileOperationUpdateAttrs: acp = new(AttributesChange) - } if acp == nil { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 7fa347854..4f0544250 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -24,6 +24,7 @@ type BaseFileChanger struct { ActualThumbnailSize int64 `json:"actual_thumb_size"` //client side: ActualThumbnailHash string `json:"actual_thumb_hash"` + //client side: MimeType string `json:"mimetype,omitempty"` //client side: diff --git a/code/go/0chain.net/blobbercore/handler/file_command_add.go b/code/go/0chain.net/blobbercore/handler/file_command_add.go index 85abecb97..3907b6309 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_add.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_add.go @@ -157,6 +157,10 @@ func (cmd *AddFileCommand) reloadChange(connectionObj *allocation.AllocationChan } cmd.fileChanger.Size = dbChangeProcessor.Size + cmd.fileChanger.ThumbnailFilename = dbChangeProcessor.ThumbnailFilename + cmd.fileChanger.ThumbnailSize = dbChangeProcessor.ThumbnailSize + cmd.fileChanger.ThumbnailHash = dbChangeProcessor.Hash + return } } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_insert.go b/code/go/0chain.net/blobbercore/handler/file_command_insert.go deleted file mode 100644 index 16701bf1a..000000000 --- a/code/go/0chain.net/blobbercore/handler/file_command_insert.go +++ /dev/null @@ -1,135 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "net/http" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/zboxcore/fileref" -) - -// InsertFileCommand command for inserting file -type InsertFileCommand struct { - allocationChange *allocation.AllocationChange - fileChanger *allocation.UpdateFileChanger -} - -// IsAuthorized validate request. -func (cmd *InsertFileCommand) IsAuthorized(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { - - if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { - return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") - } - - fileChanger := &allocation.UpdateFileChanger{} - - uploadMetaString := req.FormValue("uploadMeta") - err := json.Unmarshal([]byte(uploadMetaString), fileChanger) - if err != nil { - return common.NewError("invalid_parameters", - "Invalid parameters. Error parsing the meta data for upload."+err.Error()) - } - exisitingFileRef, _ := reference.GetReference(ctx, allocationObj.ID, fileChanger.Path) - - if exisitingFileRef != nil { - return common.NewError("duplicate_file", "File at path already exists") - } - - if fileChanger.ChunkSize <= 0 { - fileChanger.ChunkSize = fileref.CHUNK_SIZE - } - - cmd.fileChanger = fileChanger - - return nil -} - -// ProcessContent flush file to FileStorage -func (cmd *InsertFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { - - result := blobberhttp.UploadResult{} - result.Filename = cmd.fileChanger.Filename - - origfile, _, err := req.FormFile("uploadFile") - if err != nil { - return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) - } - defer origfile.Close() - - fileInputData := &filestore.FileInputData{Name: cmd.fileChanger.Filename, Path: cmd.fileChanger.Path, OnCloud: false} - fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, fileInputData, origfile, connectionObj.ConnectionID) - if err != nil { - return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) - } - - result.Hash = fileOutputData.ContentHash - result.MerkleRoot = fileOutputData.MerkleRoot - result.Size = fileOutputData.Size - - if len(cmd.fileChanger.Hash) > 0 && cmd.fileChanger.Hash != fileOutputData.ContentHash { - return result, common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the file content") - } - if len(cmd.fileChanger.MerkleRoot) > 0 && cmd.fileChanger.MerkleRoot != fileOutputData.MerkleRoot { - return result, common.NewError("content_merkle_root_mismatch", "Merkle root provided in the meta data does not match the file content") - } - - cmd.fileChanger.Hash = fileOutputData.ContentHash - cmd.fileChanger.MerkleRoot = fileOutputData.MerkleRoot - cmd.fileChanger.AllocationID = allocationObj.ID - cmd.fileChanger.Size = fileOutputData.Size - - allocationSize := fileOutputData.Size - - if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize { - return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") - } - - cmd.allocationChange = &allocation.AllocationChange{} - cmd.allocationChange.ConnectionID = connectionObj.ConnectionID - cmd.allocationChange.Size = allocationSize - cmd.allocationChange.Operation = constants.FileOperationInsert - - connectionObj.Size += cmd.allocationChange.Size - - return result, nil - -} - -// ProcessThumbnail flush thumbnail file to FileStorage if it has. -func (cmd *InsertFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { - - thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") - - if thumbHeader != nil { - - defer thumbfile.Close() - - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} - thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, thumbInputData, thumbfile, connectionObj.ConnectionID) - if err != nil { - return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) - } - if len(cmd.fileChanger.ThumbnailHash) > 0 && cmd.fileChanger.ThumbnailHash != thumbOutputData.ContentHash { - return common.NewError("content_hash_mismatch", "Content hash provided in the meta data does not match the thumbnail content") - } - - cmd.fileChanger.ThumbnailHash = thumbOutputData.ContentHash - cmd.fileChanger.ThumbnailSize = thumbOutputData.Size - cmd.fileChanger.ThumbnailFilename = thumbInputData.Name - } - - return nil - -} - -// UpdateChange add NewFileChange in db -func (cmd *InsertFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { - connectionObj.AddChange(cmd.allocationChange, cmd.fileChanger) - return connectionObj.Save(ctx) -} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index f3add4174..5149a5919 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -158,7 +158,7 @@ func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationC for _, c := range connectionObj.Changes { if c.Operation == constants.FileOperationUpdate { - dbFileChanger := &allocation.AddFileChanger{} + dbFileChanger := &allocation.UpdateFileChanger{} err := dbFileChanger.Unmarshal(c.Input) if err != nil { @@ -167,6 +167,9 @@ func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationC // reload uploaded size from db, it was chunk size from client cmd.fileChanger.Size = dbFileChanger.Size + cmd.fileChanger.ThumbnailFilename = dbFileChanger.ThumbnailFilename + cmd.fileChanger.ThumbnailSize = dbFileChanger.ThumbnailSize + cmd.fileChanger.ThumbnailHash = dbFileChanger.Hash return } } From e8bb202b0ca19867794ec282448fe2aaae077d8b Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 29 Jan 2022 00:15:24 +0800 Subject: [PATCH 094/161] fix(hash): updated sha1 with sha256 for file hash (#515) * fix(hash): updated sha1 with sha256 for file hash --- .github/workflows/benchmark.yml | 2 +- code/go/0chain.net/blobbercore/filestore/fs_store.go | 8 ++++---- code/go/0chain.net/blobbercore/filestore/mock_store.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d815d4c30..7a3c20484 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,4 +1,4 @@ -name: Benchmark tests +name: Benchmark concurrency: group: "benchmark-${{ github.ref }}" diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index 0dc6c2572..33d0b9f3c 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -3,7 +3,7 @@ package filestore import ( "bytes" "context" - "crypto/sha1" + "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -475,7 +475,7 @@ func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, fileRef := &FileOutputData{} - h := sha1.New() + h := sha256.New() bytesBuffer := bytes.NewBuffer(nil) multiHashWriter := io.MultiWriter(h, bytesBuffer) tReader := io.TeeReader(infile, multiHashWriter) @@ -548,7 +548,7 @@ func (fs *FileFSStore) WriteChunk(allocationID string, fileData *FileInputData, fileRef.ChunkUploaded = true } - h := sha1.New() + h := sha256.New() size, err := dest.WriteChunk(context.TODO(), fileData.UploadOffset, io.TeeReader(infile, h)) if err != nil { @@ -576,7 +576,7 @@ func (fs *FileFSStore) IterateObjects(allocationID string, handler FileObjectHan return nil } defer f.Close() - h := sha1.New() + h := sha256.New() if _, err := io.Copy(h, f); err != nil { return nil } diff --git a/code/go/0chain.net/blobbercore/filestore/mock_store.go b/code/go/0chain.net/blobbercore/filestore/mock_store.go index fcda9fc75..1b87ff6af 100644 --- a/code/go/0chain.net/blobbercore/filestore/mock_store.go +++ b/code/go/0chain.net/blobbercore/filestore/mock_store.go @@ -1,7 +1,7 @@ package filestore import ( - "crypto/sha1" + "crypto/sha256" "encoding/hex" "encoding/json" "io" @@ -31,7 +31,7 @@ func (ms *MockStore) WriteFile(allocationID string, fileData *FileInputData, inf fileRef.ChunkUploaded = true - h := sha1.New() + h := sha256.New() reader := io.TeeReader(infile, h) fileSize := int64(0) for { From 6daf97257f66c0663a0b1873ac6c1ba7385997a4 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Mon, 31 Jan 2022 10:49:56 +0545 Subject: [PATCH 095/161] Put spread out sql in one place (#519) --- sql/00-create-user.sql | 1 + sql/01-create-table.sql | 89 ++++++++++++++- sql/02-create-indexes.sql | 35 ++++++ sql/03-add-allocation-prices.sql | 104 ------------------ ...grant-priv.sql => 03-grant-privileges.sql} | 0 sql/04-add-on-cloud.sql | 3 - sql/05-add-commit-meta-txns-table.sql | 7 -- ...6-add-cleaned_up-column-to-allocations.sql | 7 -- sql/07-terms-belongs-to-allocation-id.sql | 30 ----- sql/08-add-payer-id-to-allocations.sql | 4 - ...10-add-time_unit-column-to-allocations.sql | 11 -- ...and-auth_tiket-columns-to-read_markers.sql | 11 -- ...attributes-column-to-reference_objects.sql | 12 -- sql/13-add-collaborators-table.sql | 10 -- sql/14-increase_owner_pubkey.sql | 16 --- sql/15-add-allocation-columns.sql | 6 - ...15-add-chunk_size-to-reference_objects.sql | 9 -- sql/16-add-marketplace-table.sql | 23 ---- sql/17-add-indexes-to-reference-objects.sql | 4 - ...column-to-read-markers.sql => comment.txt} | 24 ++-- sql/create-indexes.sql | 8 -- 21 files changed, 135 insertions(+), 279 deletions(-) create mode 100644 sql/02-create-indexes.sql delete mode 100644 sql/03-add-allocation-prices.sql rename sql/{02-grant-priv.sql => 03-grant-privileges.sql} (100%) delete mode 100644 sql/04-add-on-cloud.sql delete mode 100644 sql/05-add-commit-meta-txns-table.sql delete mode 100644 sql/06-add-cleaned_up-column-to-allocations.sql delete mode 100644 sql/07-terms-belongs-to-allocation-id.sql delete mode 100644 sql/08-add-payer-id-to-allocations.sql delete mode 100644 sql/10-add-time_unit-column-to-allocations.sql delete mode 100644 sql/11-add-payer_id-and-auth_tiket-columns-to-read_markers.sql delete mode 100644 sql/12-add-attributes-column-to-reference_objects.sql delete mode 100644 sql/13-add-collaborators-table.sql delete mode 100644 sql/14-increase_owner_pubkey.sql delete mode 100644 sql/15-add-allocation-columns.sql delete mode 100644 sql/15-add-chunk_size-to-reference_objects.sql delete mode 100644 sql/16-add-marketplace-table.sql delete mode 100644 sql/17-add-indexes-to-reference-objects.sql rename sql/{09-add-suspend-column-to-read-markers.sql => comment.txt} (76%) delete mode 100644 sql/create-indexes.sql diff --git a/sql/00-create-user.sql b/sql/00-create-user.sql index d3a0a16eb..50d7988a8 100644 --- a/sql/00-create-user.sql +++ b/sql/00-create-user.sql @@ -1,5 +1,6 @@ CREATE extension ltree; CREATE DATABASE blobber_meta; + \connect blobber_meta; CREATE USER blobber_user WITH ENCRYPTED PASSWORD 'blobber'; GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user; \ No newline at end of file diff --git a/sql/01-create-table.sql b/sql/01-create-table.sql index b8426aad1..4f8fdc787 100644 --- a/sql/01-create-table.sql +++ b/sql/01-create-table.sql @@ -11,16 +11,23 @@ $$ language 'plpgsql'; CREATE TABLE allocations( id VARCHAR (64) PRIMARY KEY, + tx VARCHAR (64) NOT NULL, size BIGINT NOT NULL DEFAULT 0, used_size BIGINT NOT NULL DEFAULT 0, owner_id VARCHAR(64) NOT NULL, - owner_public_key VARCHAR(256) NOT NULL, + payer_id VARCHAR(64) NOT NULL, + repairer_id VARCHAR(64) NOT NULL, + owner_public_key VARCHAR(512) NOT NULL, expiration_date BIGINT NOT NULL, allocation_root VARCHAR(255) NOT NULL DEFAULT '', blobber_size BIGINT NOT NULL DEFAULT 0, blobber_size_used BIGINT NOT NULL DEFAULT 0, latest_redeemed_write_marker VARCHAR(255), is_redeem_required BOOLEAN, + is_immutable BOOLEAN NOT NULL, + cleaned_up BOOLEAN NOT NULL DEFAULT FALSE, + finalized BOOLEAN NOT NULL DEFAULT FALSE, + time_unit BIGINT NOT NULL DEFAULT 172800000000000, created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -52,6 +59,49 @@ CREATE TABLE allocation_changes( CREATE TRIGGER allocation_changes_modtime BEFORE UPDATE ON allocation_changes FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); +CREATE TABLE terms ( + id bigserial, + blobber_id varchar(64) NOT NULL, + allocation_id varchar(64) REFERENCES allocations (id), + read_price bigint NOT NULL, + write_price bigint NOT NULL, + + PRIMARY KEY (id) +); + +-- clients' pending reads / writes +CREATE TABLE pendings ( + id bigserial, + client_id varchar(64) NOT NULL, + allocation_id varchar(64) NOT NULL, + blobber_id varchar(64) NOT NULL, + pending_write bigint NOT NULL DEFAULT 0, -- number of pending bytes + + PRIMARY KEY (id) +); + +CREATE TABLE read_pools ( + pool_id text NOT NULL, -- unique + client_id varchar(64) NOT NULL, + blobber_id varchar(64) NOT NULL, + allocation_id varchar(64) NOT NULL, + balance bigint NOT NULL, + expire_at bigint NOT NULL, + + PRIMARY KEY (pool_id) +); + +CREATE TABLE write_pools ( + pool_id text NOT NULL, -- unique + client_id varchar(64) NOT NULL, + blobber_id varchar(64) NOT NULL, + allocation_id varchar(64) NOT NULL, + balance bigint NOT NULL, + expire_at bigint NOT NULL, + + PRIMARY KEY (pool_id) +); + CREATE TABLE reference_objects ( id BIGSERIAL PRIMARY KEY, lookup_hash VARCHAR (64) NOT NULL, @@ -67,6 +117,7 @@ CREATE TABLE reference_objects ( custom_meta TEXT NOT NULL, content_hash VARCHAR(64) NOT NULL, size BIGINT NOT NULL DEFAULT 0, + chunk_size INT NOT NULL DEFAULT 65536, merkle_root VARCHAR(64) NOT NULL, actual_file_size BIGINT NOT NULL DEFAULT 0, actual_file_hash VARCHAR(64) NOT NULL, @@ -77,6 +128,8 @@ CREATE TABLE reference_objects ( actual_thumbnail_size BIGINT NOT NULL DEFAULT 0, actual_thumbnail_hash VARCHAR(64) NOT NULL, encrypted_key TEXT, + attributes JSON DEFAULT '{}'::jsonb, + on_cloud BOOLEAN DEFAULT FALSE, created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW(), deleted_at TIMESTAMP @@ -98,7 +151,7 @@ CREATE TABLE write_markers ( redeem_retries INT NOT NULL DEFAULT 0, close_txn_id VARCHAR(64), connection_id VARCHAR(64) NOT NULL, - client_key VARCHAR(256) NOT NULL, + client_key VARCHAR(512) NOT NULL, sequence BIGSERIAL UNIQUE, created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW() @@ -108,12 +161,15 @@ CREATE TRIGGER write_markers_modtime BEFORE UPDATE ON write_markers FOR EACH ROW CREATE TABLE read_markers ( client_id VARCHAR(64) NOT NULL PRIMARY KEY, - client_public_key VARCHAR(256) NOT NULL, + client_public_key VARCHAR(512) NOT NULL, blobber_id VARCHAR(64) NOT NULL, allocation_id VARCHAR(64) NOT NULL, owner_id VARCHAR(64) NOT NULL, + payer_id VARCHAR(64) NOT NULL, + auth_ticket JSON, timestamp BIGINT NOT NULL, counter BIGINT NOT NULL DEFAULT 0, + suspend BIGINT NOT NULL DEFAULT -1, signature VARCHAR(256) NOT NULL, latest_redeemed_rm JSON, redeem_required boolean, @@ -163,3 +219,30 @@ CREATE TABLE file_stats ( ); CREATE TRIGGER file_stats_modtime BEFORE UPDATE ON file_stats FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); + +CREATE TABLE commit_meta_txns ( + ref_id BIGSERIAL NOT NULL, + txn_id VARCHAR(64) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE TABLE collaborators ( + ref_id BIGSERIAL NOT NULL, + client_id VARCHAR(64) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE TABLE marketplace_share_info ( + id BIGSERIAL PRIMARY KEY, + owner_id VARCHAR(64) NOT NULL, + client_id VARCHAR(64) NOT NULL, + file_path_hash TEXT NOT NULL, + re_encryption_key TEXT NOT NULL, + client_encryption_public_key TEXT NOT NULL, + expiry_at TIMESTAMP NULL, + revoked BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE TRIGGER share_info_modtime BEFORE UPDATE ON marketplace_share_info FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); diff --git a/sql/02-create-indexes.sql b/sql/02-create-indexes.sql new file mode 100644 index 000000000..daf2fecd5 --- /dev/null +++ b/sql/02-create-indexes.sql @@ -0,0 +1,35 @@ +\connect blobber_meta; +BEGIN; +DROP INDEX IF EXISTS idx_unique_allocations_tx; +CREATE UNIQUE INDEX idx_unique_allocations_tx ON allocations (tx); + +DROP INDEX IF EXISTS idx_pendings_cab; +CREATE UNIQUE INDEX idx_pendings_cab ON pendings (client_id, allocation_id, blobber_id); + +DROP INDEX IF EXISTS idx_read_pools_cab; +CREATE INDEX idx_read_pools_cab ON read_pools (client_id, allocation_id, blobber_id); + +DROP INDEX IF EXISTS idx_write_pools_cab; +CREATE INDEX idx_write_pools_cab ON write_pools (client_id, allocation_id, blobber_id); + +-- TODO one of path_idx and idx_reference_objects_for_path is redundant +-- Create index on path column; It cannot be Unique index because of soft delete by gorm +DROP INDEX IF EXISTS path_idx; +CREATE INDEX path_idx ON reference_objects (path); + +DROP INDEX IF EXISTS update_idx; +CREATE INDEX update_idx ON reference_objects (updated_at); + +DROP INDEX IF EXISTS idx_reference_objects_for_lookup_hash; +CREATE INDEX idx_reference_objects_for_lookup_hash ON reference_objects(allocation_id, lookup_hash); + +DROP INDEX IF EXISTS idx_reference_objects_for_path; +CREATE INDEX idx_reference_objects_for_path ON reference_objects(allocation_id, path); + +DROP INDEX IF EXISTS idx_marketplace_share_info_for_owner; +CREATE INDEX idx_marketplace_share_info_for_owner ON marketplace_share_info(owner_id, file_path_hash); + +DROP INDEX IF EXISTS idx_marketplace_share_info_for_client; +CREATE INDEX idx_marketplace_share_info_for_client ON marketplace_share_info(client_id, file_path_hash); + +COMMIT; \ No newline at end of file diff --git a/sql/03-add-allocation-prices.sql b/sql/03-add-allocation-prices.sql deleted file mode 100644 index 846abeacb..000000000 --- a/sql/03-add-allocation-prices.sql +++ /dev/null @@ -1,104 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE allocations ADD COLUMN tx varchar (64) NOT NULL; - -CREATE UNIQUE INDEX idx_unique_allocations_tx ON allocations (tx); - -CREATE TABLE terms ( - id bigserial, - - blobber_id varchar(64) NOT NULL, - allocation_tx varchar(64) REFERENCES allocations (tx), - - read_price bigint NOT NULL, - write_price bigint NOT NULL, - - PRIMARY KEY (id) -); - --- clients' pending reads / writes -CREATE TABLE pendings ( - id bigserial, - - client_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - - pending_read bigint NOT NULL DEFAULT 0, -- number of pending blocks - pending_write bigint NOT NULL DEFAULT 0, -- number of pending bytes - - PRIMARY KEY (id) -); - -CREATE UNIQUE INDEX idx_pendings_cab - ON pendings (client_id, allocation_id, blobber_id); - -CREATE TABLE read_pools ( - pool_id text NOT NULL, -- unique - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - balance bigint NOT NULL, - expire_at bigint NOT NULL, - - PRIMARY KEY (pool_id) -); - -CREATE UNIQUE INDEX idx_read_pools_cab - ON read_pools (client_id, allocation_id, blobber_id); - -CREATE TABLE write_pools ( - pool_id text NOT NULL, -- unique - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - balance bigint NOT NULL, - expire_at bigint NOT NULL, - - PRIMARY KEY (pool_id) -); - -CREATE UNIQUE INDEX idx_write_pools_cab - ON write_pools (client_id, allocation_id, blobber_id); - - -CREATE TABLE read_redeems ( - id bigserial, - - read_counter bigint NOT NULL, - value bigint NOT NULL, - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - PRIMARY KEY (id) -); - -CREATE TABLE write_redeems ( - id bigserial, - - signature varchar(256) NOT NULL, -- write marker signature - - size bigint NOT NULL, - value bigint NOT NULL, - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - PRIMARY KEY (id) -); - -CREATE INDEX idx_write_redeems_signature ON write_redeems (signature); - --- --- grant again --- - -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO blobber_user; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO blobber_user; diff --git a/sql/02-grant-priv.sql b/sql/03-grant-privileges.sql similarity index 100% rename from sql/02-grant-priv.sql rename to sql/03-grant-privileges.sql diff --git a/sql/04-add-on-cloud.sql b/sql/04-add-on-cloud.sql deleted file mode 100644 index 4ad182657..000000000 --- a/sql/04-add-on-cloud.sql +++ /dev/null @@ -1,3 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE reference_objects ADD COLUMN on_cloud BOOLEAN DEFAULT FALSE; \ No newline at end of file diff --git a/sql/05-add-commit-meta-txns-table.sql b/sql/05-add-commit-meta-txns-table.sql deleted file mode 100644 index 9560bde5b..000000000 --- a/sql/05-add-commit-meta-txns-table.sql +++ /dev/null @@ -1,7 +0,0 @@ -\connect blobber_meta; - -CREATE TABLE commit_meta_txns ( - ref_id BIGSERIAL NOT NULL, - txn_id VARCHAR(64) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW() -); \ No newline at end of file diff --git a/sql/06-add-cleaned_up-column-to-allocations.sql b/sql/06-add-cleaned_up-column-to-allocations.sql deleted file mode 100644 index b6551a670..000000000 --- a/sql/06-add-cleaned_up-column-to-allocations.sql +++ /dev/null @@ -1,7 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE allocations - ADD COLUMN cleaned_up boolean NOT NULL DEFAULT false; - -ALTER TABLE allocations - ADD COLUMN finalized boolean NOT NULL DEFAULT false; diff --git a/sql/07-terms-belongs-to-allocation-id.sql b/sql/07-terms-belongs-to-allocation-id.sql deleted file mode 100644 index 4080ae32b..000000000 --- a/sql/07-terms-belongs-to-allocation-id.sql +++ /dev/null @@ -1,30 +0,0 @@ -\connect blobber_meta; - -BEGIN; - - ALTER TABLE terms - ADD COLUMN allocation_id varchar(64) REFERENCES allocations (id); - - UPDATE terms AS t - SET allocation_id = a.id - FROM allocations AS a - WHERE t.allocation_tx = a.tx; - - ALTER TABLE terms DROP COLUMN allocation_tx; - -COMMIT; - -BEGIN; - -- drop unique index - DROP INDEX idx_read_pools_cab; - DROP INDEX idx_write_pools_cab; - - -- create non-unique - CREATE INDEX idx_read_pools_cab - ON read_pools (client_id, allocation_id, blobber_id); - CREATE INDEX idx_write_pools_cab - ON write_pools (client_id, allocation_id, blobber_id); -COMMIT; - --- for the commit_meta_txns -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO blobber_user; diff --git a/sql/08-add-payer-id-to-allocations.sql b/sql/08-add-payer-id-to-allocations.sql deleted file mode 100644 index 525a97c12..000000000 --- a/sql/08-add-payer-id-to-allocations.sql +++ /dev/null @@ -1,4 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE allocations ADD COLUMN payer_id VARCHAR(64) NOT NULL; - diff --git a/sql/10-add-time_unit-column-to-allocations.sql b/sql/10-add-time_unit-column-to-allocations.sql deleted file mode 100644 index 49a4e6d37..000000000 --- a/sql/10-add-time_unit-column-to-allocations.sql +++ /dev/null @@ -1,11 +0,0 @@ --- --- Add column time_unit to allocations. Default is 48h. --- - --- pew-pew -\connect blobber_meta; - -BEGIN; - ALTER TABLE allocations - ADD COLUMN time_unit BIGINT NOT NULL DEFAULT 172800000000000; -COMMIT; diff --git a/sql/11-add-payer_id-and-auth_tiket-columns-to-read_markers.sql b/sql/11-add-payer_id-and-auth_tiket-columns-to-read_markers.sql deleted file mode 100644 index af7865ea8..000000000 --- a/sql/11-add-payer_id-and-auth_tiket-columns-to-read_markers.sql +++ /dev/null @@ -1,11 +0,0 @@ --- --- add payer_id and auth_tiket columns to read_markers table --- - --- pew-pew -\connect blobber_meta; - -BEGIN; - ALTER TABLE read_markers ADD COLUMN payer_id VARCHAR(64) NOT NULL; - ALTER TABLE read_markers ADD COLUMN auth_ticket JSON; -COMMIT; diff --git a/sql/12-add-attributes-column-to-reference_objects.sql b/sql/12-add-attributes-column-to-reference_objects.sql deleted file mode 100644 index 47db2e20c..000000000 --- a/sql/12-add-attributes-column-to-reference_objects.sql +++ /dev/null @@ -1,12 +0,0 @@ --- --- Add who_pays column to reference_objects table. --- - --- pew-pew -\connect blobber_meta; - --- in a transaction -BEGIN; - ALTER TABLE reference_objects - ADD COLUMN attributes JSON DEFAULT '{}'::jsonb; -COMMIT; diff --git a/sql/13-add-collaborators-table.sql b/sql/13-add-collaborators-table.sql deleted file mode 100644 index 9dec684fe..000000000 --- a/sql/13-add-collaborators-table.sql +++ /dev/null @@ -1,10 +0,0 @@ -\connect blobber_meta; - - -CREATE TABLE collaborators ( - ref_id BIGSERIAL NOT NULL, - client_id VARCHAR(64) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW() -); - -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO blobber_user; \ No newline at end of file diff --git a/sql/14-increase_owner_pubkey.sql b/sql/14-increase_owner_pubkey.sql deleted file mode 100644 index f6cd40520..000000000 --- a/sql/14-increase_owner_pubkey.sql +++ /dev/null @@ -1,16 +0,0 @@ --- --- Increase the char limit of owner_public_key from 256 to 512. --- - --- pew-pew -\connect blobber_meta; - --- in a transaction -BEGIN; - ALTER TABLE allocations - ALTER COLUMN owner_public_key TYPE varchar(512); - ALTER TABLE read_markers - ALTER COLUMN client_public_key TYPE varchar(512); - ALTER TABLE write_markers - ALTER COLUMN client_key TYPE varchar(512); -COMMIT; diff --git a/sql/15-add-allocation-columns.sql b/sql/15-add-allocation-columns.sql deleted file mode 100644 index c983cf73f..000000000 --- a/sql/15-add-allocation-columns.sql +++ /dev/null @@ -1,6 +0,0 @@ -\connect blobber_meta; - -BEGIN; - ALTER TABLE allocations ADD COLUMN repairer_id VARCHAR(64) NOT NULL; - ALTER TABLE allocations ADD COLUMN is_immutable BOOLEAN NOT NULL; -COMMIT; \ No newline at end of file diff --git a/sql/15-add-chunk_size-to-reference_objects.sql b/sql/15-add-chunk_size-to-reference_objects.sql deleted file mode 100644 index a2530e3f0..000000000 --- a/sql/15-add-chunk_size-to-reference_objects.sql +++ /dev/null @@ -1,9 +0,0 @@ --- --- Add chunk_size column to reference_objects table. --- - --- pew-pew -\connect blobber_meta; - - -ALTER TABLE reference_objects ADD COLUMN chunk_size INT NOT NULL DEFAULT 65536; diff --git a/sql/16-add-marketplace-table.sql b/sql/16-add-marketplace-table.sql deleted file mode 100644 index bbfd4b043..000000000 --- a/sql/16-add-marketplace-table.sql +++ /dev/null @@ -1,23 +0,0 @@ -\connect blobber_meta; - - -CREATE TABLE marketplace_share_info ( - id BIGSERIAL PRIMARY KEY, - owner_id VARCHAR(64) NOT NULL, - client_id VARCHAR(64) NOT NULL, - file_path_hash TEXT NOT NULL, - re_encryption_key TEXT NOT NULL, - client_encryption_public_key TEXT NOT NULL, - expiry_at TIMESTAMP NULL, - revoked BOOLEAN NOT NULL DEFAULT false, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_marketplace_share_info_for_owner ON marketplace_share_info(owner_id, file_path_hash); -CREATE INDEX idx_marketplace_share_info_for_client ON marketplace_share_info(client_id, file_path_hash); - -CREATE TRIGGER share_info_modtime BEFORE UPDATE ON marketplace_share_info FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); - -GRANT ALL PRIVILEGES ON TABLE marketplace_share_info TO blobber_user; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO blobber_user; diff --git a/sql/17-add-indexes-to-reference-objects.sql b/sql/17-add-indexes-to-reference-objects.sql deleted file mode 100644 index dc8146e84..000000000 --- a/sql/17-add-indexes-to-reference-objects.sql +++ /dev/null @@ -1,4 +0,0 @@ -\connect blobber_meta; - -CREATE INDEX idx_reference_objects_for_lookup_hash ON reference_objects(allocation_id, lookup_hash); -CREATE INDEX idx_reference_objects_for_path ON reference_objects(allocation_id, path); diff --git a/sql/09-add-suspend-column-to-read-markers.sql b/sql/comment.txt similarity index 76% rename from sql/09-add-suspend-column-to-read-markers.sql rename to sql/comment.txt index 39d8e3ec8..5944b8d2f 100644 --- a/sql/09-add-suspend-column-to-read-markers.sql +++ b/sql/comment.txt @@ -1,6 +1,6 @@ -\connect blobber_meta; +-- \connect blobber_meta; -BEGIN; +-- BEGIN; -- if the suspend is equal to 'counter' column, then we don't send the marker -- even if its 'redeem_required' column is set to true; for a case, where we @@ -17,15 +17,16 @@ BEGIN; -- expired -- - ALTER TABLE read_markers - ADD COLUMN suspend BIGINT NOT NULL DEFAULT -1; + -- Changes have been moved to table creation + -- ALTER TABLE read_markers + -- ADD COLUMN suspend BIGINT NOT NULL DEFAULT -1; -- -- we don't need to track pending reads anymore -- - - ALTER TABLE pendings - DROP COLUMN pending_read; + -- Changes have been moved to table creation + -- ALTER TABLE pendings + -- DROP COLUMN pending_read; -- -- pending values has changed from tokens to number of block for read markers @@ -39,7 +40,7 @@ BEGIN; -- the transaction; thus the pending will not be reset next time, that's -- expected) - UPDATE pendings SET pending_write = 0; + -- UPDATE pendings SET pending_write = 0 -- -- don't track every redeem, since update allocation or slow redeeming can @@ -47,7 +48,8 @@ BEGIN; -- to track pending redeems -- - DROP TABLE read_redeems CASCADE; - DROP TABLE write_redeems CASCADE; -- with indices + -- Changes have been moved to table creation; Table is simply never created + -- DROP TABLE read_redeems CASCADE; + -- DROP TABLE write_redeems CASCADE; -- with indices -COMMIT; +-- COMMIT; diff --git a/sql/create-indexes.sql b/sql/create-indexes.sql deleted file mode 100644 index f0d57dbe6..000000000 --- a/sql/create-indexes.sql +++ /dev/null @@ -1,8 +0,0 @@ -\connect blobber_meta; -BEGIN; -DROP INDEX IF EXISTS path_idx; -DROP INDEX IF EXISTS update_idx; --- Create index on path column; It cannot be Unique index because of soft delete by gorm -CREATE INDEX path_idx ON reference_objects (path); -CREATE INDEX update_idx ON reference_objects (updated_at); -COMMIT; \ No newline at end of file From 6041fff4855a9ffee008b4bf8ec5e6ff5b7b3caa Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Thu, 3 Feb 2022 09:24:47 +0545 Subject: [PATCH 096/161] Fix issue raised by linter (#520) * Fix issue raised by gocritic linter * Fix issue raised by goimports linter * Fix issue raised by gosec linter * Fix issue raised by unconvert linter * Fix issue raised by unparam linter * Fix issue raised by misspell linter * Fix issue raised by whitespace linter --- code/go/0chain.net/blobber/datastore.go | 2 - code/go/0chain.net/blobber/flags.go | 1 - code/go/0chain.net/blobber/grpc.go | 1 - code/go/0chain.net/blobber/http.go | 2 - code/go/0chain.net/blobber/main.go | 1 - code/go/0chain.net/blobber/node.go | 1 - code/go/0chain.net/blobber/zcn.go | 3 - .../allocation/allocationchange.go | 3 +- .../allocation/attributesfilechange.go | 4 +- .../blobbercore/allocation/copyfilechange.go | 2 +- .../allocation/deletefilechange.go | 1 - .../blobbercore/allocation/entity.go | 38 ++----- .../allocation/file_changer_add.go | 4 +- .../allocation/file_changer_update.go | 1 - .../blobbercore/allocation/newfilechange.go | 4 +- .../blobbercore/allocation/protocol.go | 31 +++--- .../allocation/renamefilechange_test.go | 2 +- .../blobbercore/allocation/workers.go | 30 ++---- .../blobbercore/blobberhttp/response.go | 4 +- .../blobbercore/challenge/challenge.go | 9 +- .../blobbercore/challenge/entity.go | 4 +- .../blobbercore/challenge/protocol.go | 4 +- .../blobbercore/challenge/worker.go | 2 - .../0chain.net/blobbercore/config/config.go | 3 +- .../0chain.net/blobbercore/convert/convert.go | 3 +- .../blobbercore/convert/response_creator.go | 7 +- .../blobbercore/datastore/mock_store.go | 1 - .../blobbercore/datastore/mocket.go | 2 - .../blobbercore/datastore/sqlmock.go | 2 - .../blobbercore/filestore/chunk_writer.go | 4 +- .../filestore/chunk_writer_test.go | 4 +- .../blobbercore/filestore/fs_store.go | 32 +++--- .../blobbercore/filestore/mock_store.go | 6 +- .../calculate_hash_integration_test.go | 3 +- .../handler/collaborator_integration_test.go | 3 +- .../commit_meta_txn_integration_test.go | 3 +- .../handler/copy_object_integration_test.go | 3 +- .../blobbercore/handler/file_command.go | 2 +- .../blobbercore/handler/file_command_add.go | 49 +++++---- .../handler/file_command_delete.go | 4 +- .../handler/file_command_update.go | 52 +++++---- .../handler/getallocation_integration_test.go | 1 - .../getfilemetadata_integration_test.go | 3 +- .../handler/getfilestats_integration_test.go | 1 - .../handler/getobjecttree_integration_test.go | 1 - .../getreferencepath_integration_test.go | 1 - .../blobbercore/handler/grpc_handler.go | 23 ++-- .../blobbercore/handler/grpc_handler_test.go | 5 +- .../blobbercore/handler/grpc_middleware.go | 2 +- .../0chain.net/blobbercore/handler/handler.go | 7 +- .../blobbercore/handler/handler_test.go | 40 ++++--- .../handler/helper_integration_test.go | 7 +- .../handler/listentities_integration_test.go | 3 +- .../handler/object_operation_grpc_handler.go | 10 +- .../handler/object_operation_handler.go | 102 ++++++++---------- .../object_operation_handler_bench_test.go | 9 -- .../handler/object_operation_handler_test.go | 1 - .../blobbercore/handler/protocol.go | 4 +- .../handler/renameobject_integration_test.go | 3 +- .../blobbercore/handler/storage_handler.go | 84 +++++++-------- .../updateattributes_integration_test.go | 3 +- .../0chain.net/blobbercore/handler/worker.go | 7 +- .../0chain.net/blobbercore/handler/zcncore.go | 3 +- .../0chain.net/blobbercore/mock/allocation.go | 2 +- code/go/0chain.net/blobbercore/mock/ctx.go | 1 - code/go/0chain.net/blobbercore/mock/init.go | 1 - .../blobbercore/readmarker/entity.go | 10 +- .../blobbercore/readmarker/protocol.go | 16 +-- .../blobbercore/readmarker/readmarker.go | 3 +- .../blobbercore/readmarker/worker.go | 8 +- .../blobbercore/reference/object.go | 8 +- .../blobbercore/reference/objectpath.go | 6 +- .../0chain.net/blobbercore/reference/ref.go | 41 ++++--- .../blobbercore/reference/referencepath.go | 12 +-- .../blobbercore/reference/shareinfo.go | 4 +- .../blobbercore/stats/allocationstats.go | 3 +- .../blobbercore/stats/blobberstats.go | 14 +-- .../blobbercore/stats/challengestats.go | 10 +- .../0chain.net/blobbercore/stats/filestats.go | 3 +- .../0chain.net/blobbercore/stats/handler.go | 19 ++-- .../blobbercore/stats/pagination.go | 2 +- code/go/0chain.net/blobbercore/util/json.go | 7 +- .../blobbercore/writemarker/entity.go | 9 +- .../blobbercore/writemarker/protocol.go | 5 +- .../blobbercore/writemarker/worker.go | 4 +- .../0chain.net/conductor/conductrpc/entity.go | 1 - .../0chain.net/conductor/conductrpc/state.go | 1 - code/go/0chain.net/core/build/info.go | 2 +- code/go/0chain.net/core/cache/lfu.go | 6 +- code/go/0chain.net/core/cache/lru.go | 8 +- code/go/0chain.net/core/common/context.go | 6 +- code/go/0chain.net/core/common/errors.go | 4 +- code/go/0chain.net/core/common/handler.go | 32 ++---- .../go/0chain.net/core/common/rate_limiter.go | 1 - code/go/0chain.net/core/common/time.go | 2 +- code/go/0chain.net/core/common/utils.go | 12 +-- code/go/0chain.net/core/encryption/keys.go | 6 +- code/go/0chain.net/core/lock/lock.go | 6 +- code/go/0chain.net/core/lock/lock_test.go | 3 - code/go/0chain.net/core/logging/logger.go | 3 +- code/go/0chain.net/core/node/self_node.go | 2 +- code/go/0chain.net/core/transaction/http.go | 10 +- code/go/0chain.net/core/util/http.go | 12 +-- code/go/0chain.net/validator/main.go | 1 - .../0chain.net/validatorcore/config/config.go | 2 +- .../storage/challenge_handler.go | 1 - .../validatorcore/storage/models.go | 29 ++--- .../validatorcore/storage/protocol.go | 5 +- .../storage/writemarker/entity.go | 2 +- 109 files changed, 390 insertions(+), 612 deletions(-) diff --git a/code/go/0chain.net/blobber/datastore.go b/code/go/0chain.net/blobber/datastore.go index 5402cfdba..acb0e0506 100644 --- a/code/go/0chain.net/blobber/datastore.go +++ b/code/go/0chain.net/blobber/datastore.go @@ -12,7 +12,6 @@ func setupDatabase() error { fmt.Print("\r[7/11] connect data store") // check for database connection for i := 0; i < 600; i++ { - if i > 0 { fmt.Printf("\r[7/10] connect(%v) data store", i) } @@ -27,7 +26,6 @@ func setupDatabase() error { } time.Sleep(1 * time.Second) - } return nil diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go index 436c23467..f41af64e9 100644 --- a/code/go/0chain.net/blobber/flags.go +++ b/code/go/0chain.net/blobber/flags.go @@ -25,7 +25,6 @@ var ( ) func init() { - flag.IntVar(&deploymentMode, "deployment_mode", 2, "deployment mode: 0=dev,1=test, 2=mainnet") flag.StringVar(&keysFile, "keys_file", "", "keys_file") flag.StringVar(&minioFile, "minio_file", "", "minio_file") diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go index 4da0db458..cdb6ea692 100644 --- a/code/go/0chain.net/blobber/grpc.go +++ b/code/go/0chain.net/blobber/grpc.go @@ -18,7 +18,6 @@ import ( ) func startGRPCServer() { - common.ConfigRateLimits() r := mux.NewRouter() initHandlers(r) diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index 4f634e903..a946ebff9 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -18,7 +18,6 @@ import ( ) func startHttpServer() { - mode := "main net" if config.Development() { mode = "development" @@ -42,7 +41,6 @@ func startHttpServer() { fmt.Println("[11/11] start http server [OK]") wg.Wait() - } func startServer(wg *sync.WaitGroup, r *mux.Router, mode string, port int, isTls bool) { diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index c6a2455bb..2c5492f32 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -46,5 +46,4 @@ func main() { startGRPCServer() startHttpServer() - } diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index c2322903c..072e47241 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -16,7 +16,6 @@ func setupNode() error { reader, err := os.Open(keysFile) if err != nil { - return err } defer reader.Close() diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index 3c6cb4edd..896635ee0 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -45,13 +45,11 @@ func setupOnChain() { fmt.Print(" [SKIP]\n") break } else { - if err := registerBlobberOnChain(); err != nil { if i == 10 { // no more attempts panic(err) } fmt.Print("\n ", err.Error()+"\n") - } else { fmt.Print(" [OK]\n") break @@ -62,7 +60,6 @@ func setupOnChain() { fmt.Printf("\r - wait %v seconds to retry", ATTEMPT_DELAY-n) } } - } if !isIntegrationTest { go setupWorkers() diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 7ff70c93b..aecd93a52 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -69,7 +69,7 @@ func (change *AllocationChange) Save(ctx context.Context) error { // GetAllocationChanges reload connection's changes in allocation from postgres. // 1. update connection's status with NewConnection if connection_id is not found in postgres // 2. mark as NewConnection if connection_id is marked as DeleteConnection -func GetAllocationChanges(ctx context.Context, connectionID string, allocationID string, clientID string) (*AllocationChangeCollector, error) { +func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clientID string) (*AllocationChangeCollector, error) { cc := &AllocationChangeCollector{} db := datastore.GetStore().GetTransaction(ctx) err := db.Where("connection_id = ? and allocation_id = ? and client_id = ? and status <> ?", @@ -102,7 +102,6 @@ func (cc *AllocationChangeCollector) AddChange(allocationChange *AllocationChang } func (cc *AllocationChangeCollector) Save(ctx context.Context) error { - db := datastore.GetStore().GetTransaction(ctx) if cc.Status == NewConnection { cc.Status = InProgressConnection diff --git a/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go b/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go index 6b2860aab..aeb57be64 100644 --- a/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go @@ -22,9 +22,7 @@ type AttributesChange struct { } // ProcessChange processes the attributes changes. -func (ac *AttributesChange) ProcessChange(ctx context.Context, - _ *AllocationChange, allocRoot string) (ref *reference.Ref, err error) { - +func (ac *AttributesChange) ProcessChange(ctx context.Context, _ *AllocationChange, allocRoot string) (ref *reference.Ref, err error) { var path, _ = filepath.Split(ac.Path) path = filepath.Clean(path) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 9558dd834..41e4904e6 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -99,7 +99,7 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC return rootRef, err } -func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef *reference.Ref, destRef *reference.Ref, allocationRoot string) { +func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, destRef *reference.Ref, allocationRoot string) { if affectedRef.Type == reference.DIRECTORY { newRef := reference.NewDirectoryRef() newRef.AllocationID = rf.AllocationID diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index 7844e768b..d878ffd21 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -23,7 +23,6 @@ type DeleteFileChange struct { } func (nf *DeleteFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { - rootRef, contentHash, err := reference.DeleteObject(ctx, nf.AllocationID, nf.Path) if err != nil { return nil, err diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 6196ead9b..0bf78f663 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -49,9 +49,7 @@ func (Allocation) TableName() string { // RestDurationInTimeUnits returns number (float point) of time units until // allocation ends. -func (a *Allocation) RestDurationInTimeUnits(wmt common.Timestamp) ( - rdtu float64) { - +func (a *Allocation) RestDurationInTimeUnits(wmt common.Timestamp) (rdtu float64) { var ( wmtt = time.Unix(int64(wmt), 0) expt = time.Unix(int64(a.Expiration), 0) @@ -92,9 +90,7 @@ type WantWriter interface { // WantWrite returns amount of tokens (by current terms of the allocations that // should be loaded) by given size for given blobber. E.g. want is tokens // wanted. -func (a *Allocation) WantWrite(blobberID string, size int64, - wmt common.Timestamp) (value int64) { - +func (a *Allocation) WantWrite(blobberID string, size int64, wmt common.Timestamp) (value int64) { if size < 0 { return // deleting, ignore } @@ -111,9 +107,7 @@ func (a *Allocation) WantWrite(blobberID string, size int64, } // ReadPools from DB cache. -func ReadPools(tx *gorm.DB, clientID, allocID, blobberID string, - until common.Timestamp) (rps []*ReadPool, err error) { - +func ReadPools(tx *gorm.DB, clientID, allocID, blobberID string, until common.Timestamp) (rps []*ReadPool, err error) { const query = `client_id = ? AND allocation_id = ? AND blobber_id = ? AND @@ -127,9 +121,7 @@ func ReadPools(tx *gorm.DB, clientID, allocID, blobberID string, // HaveRead is sum of read pools (the list should be filtered by query // excluding pools expired and pools going to expired soon) minus pending reads. -func (a *Allocation) HaveRead(rps []*ReadPool, blobberID string, - pendNumBlocks int64) (have int64) { - +func (a *Allocation) HaveRead(rps []*ReadPool, blobberID string, pendNumBlocks int64) (have int64) { for _, rp := range rps { have += rp.Balance } @@ -150,9 +142,7 @@ func (*Pending) TableName() string { return "pendings" } -func GetPending(tx *gorm.DB, clientID, allocationID, blobberID string) ( - p *Pending, err error) { - +func GetPending(tx *gorm.DB, clientID, allocationID, blobberID string) (p *Pending, err error) { const query = `client_id = ? AND allocation_id = ? AND blobber_id = ?` @@ -180,9 +170,7 @@ func (p *Pending) SubPendingWrite(size int64) { } } -func (p *Pending) WritePools(tx *gorm.DB, blobberID string, - until common.Timestamp) (wps []*WritePool, err error) { - +func (p *Pending) WritePools(tx *gorm.DB, blobberID string, until common.Timestamp) (wps []*WritePool, err error) { const query = `client_id = ? AND allocation_id = ? AND blobber_id = ? AND @@ -194,9 +182,7 @@ func (p *Pending) WritePools(tx *gorm.DB, blobberID string, return } -func (p *Pending) HaveWrite(wps []*WritePool, ww WantWriter, - wmt common.Timestamp) (have int64) { - +func (p *Pending) HaveWrite(wps []*WritePool, ww WantWriter, wmt common.Timestamp) (have int64) { for _, wp := range wps { have += wp.Balance } @@ -254,11 +240,8 @@ func (*WritePool) TableName() string { return "write_pools" } -func SetReadPools(db *gorm.DB, clientID, allocationID, blobberID string, - rps []*ReadPool) (err error) { - +func SetReadPools(db *gorm.DB, clientID, allocationID, blobberID string, rps []*ReadPool) (err error) { // cleanup and batch insert (remove old pools, add / update new) - const query = `client_id = ? AND allocation_id = ? AND blobber_id = ?` @@ -282,9 +265,7 @@ func SetReadPools(db *gorm.DB, clientID, allocationID, blobberID string, return } -func SetWritePools(db *gorm.DB, clientID, allocationID, blobberID string, - wps []*WritePool) (err error) { - +func SetWritePools(db *gorm.DB, clientID, allocationID, blobberID string, wps []*WritePool) (err error) { const query = `client_id = ? AND allocation_id = ? AND blobber_id = ?` @@ -319,7 +300,6 @@ type ReadPoolRedeem struct { // SubReadRedeemed subtracts tokens redeemed from read pools. func SubReadRedeemed(rps []*ReadPool, redeems []ReadPoolRedeem) { - var rm = make(map[string]int64) for _, rd := range redeems { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_add.go b/code/go/0chain.net/blobbercore/allocation/file_changer_add.go index cd6dcc249..434afa0d5 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_add.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_add.go @@ -19,9 +19,7 @@ type AddFileChanger struct { } // ProcessChange update references, and create a new FileRef -func (nf *AddFileChanger) ProcessChange(ctx context.Context, - change *AllocationChange, allocationRoot string) (*reference.Ref, error) { - +func (nf *AddFileChanger) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { path, _ := filepath.Split(nf.Path) path = filepath.Clean(path) tSubDirs := reference.GetSubDirsFromPath(path) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 0119e9bb7..53e7f7fcd 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -19,7 +19,6 @@ type UpdateFileChanger struct { } func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { - path, _ := filepath.Split(nf.Path) path = filepath.Clean(path) tSubDirs := reference.GetSubDirsFromPath(path) diff --git a/code/go/0chain.net/blobbercore/allocation/newfilechange.go b/code/go/0chain.net/blobbercore/allocation/newfilechange.go index 5b48dde8c..c4e05e678 100644 --- a/code/go/0chain.net/blobbercore/allocation/newfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/newfilechange.go @@ -115,9 +115,7 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a return rootRef, nil } -func (nf *NewFileChange) ProcessChange(ctx context.Context, - change *AllocationChange, allocationRoot string) (*reference.Ref, error) { - +func (nf *NewFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { if change.Operation == constants.FileOperationCreateDir { err := nf.Unmarshal(change.Input) if err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index e4c358df9..66f928d93 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -16,9 +16,7 @@ import ( ) // GetAllocationByID from DB. This function doesn't load related terms. -func GetAllocationByID(ctx context.Context, allocID string) ( - a *Allocation, err error) { - +func GetAllocationByID(ctx context.Context, allocID string) (a *Allocation, err error) { var tx = datastore.GetStore().GetTransaction(ctx) a = new(Allocation) @@ -49,9 +47,7 @@ func (a *Allocation) LoadTerms(ctx context.Context) (err error) { } // VerifyAllocationTransaction try to get allocation from postgres.if it doesn't exists, get it from sharders, and insert it into postgres. -func VerifyAllocationTransaction(ctx context.Context, allocationTx string, - readonly bool) (a *Allocation, err error) { - +func VerifyAllocationTransaction(ctx context.Context, allocationTx string, readonly bool) (a *Allocation, err error) { var tx = datastore.GetStore().GetTransaction(ctx) a = new(Allocation) @@ -101,14 +97,15 @@ func VerifyAllocationTransaction(ctx context.Context, allocationTx string, if !isExist { foundBlobber := false for _, blobberConnection := range sa.Blobbers { - if blobberConnection.ID == node.Self.ID { - foundBlobber = true - a.AllocationRoot = "" - a.BlobberSize = (sa.Size + int64(len(sa.Blobbers)-1)) / - int64(len(sa.Blobbers)) - a.BlobberSizeUsed = 0 - break + if blobberConnection.ID != node.Self.ID { + continue } + foundBlobber = true + a.AllocationRoot = "" + a.BlobberSize = (sa.Size + int64(len(sa.Blobbers)-1)) / + int64(len(sa.Blobbers)) + a.BlobberSizeUsed = 0 + break } if !foundBlobber { return nil, common.NewError("invalid_blobber", @@ -178,9 +175,7 @@ type PoolStat struct { ExpireAt common.Timestamp `json:"expire_at"` } -func RequestReadPools(clientID, allocationID string) ( - rps []*ReadPool, err error) { - +func RequestReadPools(clientID, allocationID string) (rps []*ReadPool, err error) { Logger.Info("request read pools") var ( @@ -227,9 +222,7 @@ func RequestReadPools(clientID, allocationID string) ( return // got them } -func RequestWritePools(clientID, allocationID string) ( - wps []*WritePool, err error) { - +func RequestWritePools(clientID, allocationID string) (wps []*WritePool, err error) { Logger.Info("request write pools") var ( diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go index d1079863b..9aafd38d8 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -38,7 +38,7 @@ func (MockFileBlockGetter) GetFileBlock( fileData *filestore.FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { - return []byte(mockFileBlock), nil + return mockFileBlock, nil } func resetMockFileBlock() { diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go index b57ef59fc..bc871ee5b 100644 --- a/code/go/0chain.net/blobbercore/allocation/workers.go +++ b/code/go/0chain.net/blobbercore/allocation/workers.go @@ -73,7 +73,6 @@ func waitOrQuit(ctx context.Context, d time.Duration) (quit bool) { } func updateWork(ctx context.Context) { - defer func() { if r := recover(); r != nil { Logger.Error("[recover] updateWork", zap.Any("err", r)) @@ -91,7 +90,6 @@ func updateWork(ctx context.Context) { // iterate all in loop accepting allocations with limit for start := true; start || (offset < count); start = false { - allocs, count, err = findAllocations(ctx, offset) if err != nil { Logger.Error("finding allocations in DB", zap.Error(err)) @@ -113,9 +111,7 @@ func updateWork(ctx context.Context) { } // not finalized, not cleaned up -func findAllocations(ctx context.Context, offset int64) ( - allocs []*Allocation, count int64, err error) { - +func findAllocations(ctx context.Context, offset int64) (allocs []*Allocation, count int64, err error) { const query = `finalized = false AND cleaned_up = false` ctx = datastore.GetStore().CreateTransaction(ctx) @@ -144,7 +140,6 @@ func shouldFinalize(sa *transaction.StorageAllocation) bool { } func updateAllocation(ctx context.Context, a *Allocation) { - if a.Finalized { cleanupAllocation(ctx, a) return @@ -174,12 +169,9 @@ func updateAllocation(ctx context.Context, a *Allocation) { if a.Finalized && !a.CleanedUp { cleanupAllocation(ctx, a) } - } -func requestAllocation(allocID string) ( - sa *transaction.StorageAllocation, err error) { - +func requestAllocation(allocID string) (sa *transaction.StorageAllocation, err error) { var b []byte b, err = transaction.MakeSCRestAPICall( transaction.STORAGE_CONTRACT_ADDRESS, @@ -202,9 +194,7 @@ func commit(tx *gorm.DB, err *error) { (*err) = tx.Commit().Error } -func updateAllocationInDB(ctx context.Context, a *Allocation, - sa *transaction.StorageAllocation) (ua *Allocation, err error) { - +func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.StorageAllocation) (ua *Allocation, err error) { ctx = datastore.GetStore().CreateTransaction(ctx) var tx = datastore.GetStore().GetTransaction(ctx) @@ -234,7 +224,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, } // save allocations - if err = tx.Save(a).Error; err != nil { + if err := tx.Save(a).Error; err != nil { return nil, err } @@ -244,7 +234,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, // save allocation terms for _, t := range a.Terms { - if err = tx.Save(t).Error; err != nil { + if err := tx.Save(t).Error; err != nil { return nil, err } } @@ -265,7 +255,6 @@ func (fr *finalizeRequest) marshal() string { } func sendFinalizeAllocation(a *Allocation) { - var tx, err = transaction.NewTransactionEntity() if err != nil { Logger.Error("creating new transaction entity", zap.Error(err)) @@ -287,7 +276,6 @@ func sendFinalizeAllocation(a *Allocation) { } func cleanupAllocation(ctx context.Context, a *Allocation) { - var err error if err = deleteInFakeConnection(ctx, a); err != nil { Logger.Error("cleaning finalized allocation", zap.Error(err)) @@ -330,9 +318,7 @@ func deleteInFakeConnection(ctx context.Context, a *Allocation) (err error) { } // delete references -func deleteFiles(ctx context.Context, allocID string, - conn *AllocationChangeCollector) (err error) { - +func deleteFiles(ctx context.Context, allocID string, conn *AllocationChangeCollector) (err error) { var ( tx = datastore.GetStore().GetTransaction(ctx) refs = make([]*reference.Ref, 0) @@ -355,9 +341,7 @@ func deleteFiles(ctx context.Context, allocID string, } // delete reference -func deleteFile(ctx context.Context, path string, - conn *AllocationChangeCollector) (err error) { - +func deleteFile(ctx context.Context, path string, conn *AllocationChangeCollector) (err error) { var fileRef *reference.Ref fileRef, err = reference.GetReference(ctx, conn.AllocationID, path) if err != nil { diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index 68362fe99..1998f6b67 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -14,9 +14,9 @@ type UploadResult struct { Hash string `json:"content_hash"` MerkleRoot string `json:"merkle_root"` - //UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. + // UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. UploadLength int64 `json:"upload_length"` - //Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. + // Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. UploadOffset int64 `json:"upload_offset"` } diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index dbe081921..23f171d0f 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -43,7 +43,6 @@ func syncOpenChallenges(ctx context.Context) { if err != nil { logging.Logger.Error("[challenge]open: ", zap.Error(err)) } else { - bytesReader := bytes.NewBuffer(retBytes) d := json.NewDecoder(bytesReader) @@ -54,7 +53,7 @@ func syncOpenChallenges(ctx context.Context) { logging.Logger.Error("[challenge]json: ", zap.Error(errd)) } else { for _, challengeObj := range blobberChallenges.Challenges { - if challengeObj == nil || len(challengeObj.ChallengeID) == 0 { + if challengeObj == nil || challengeObj.ChallengeID == "" { logging.Logger.Info("[challenge]open: No challenge entity from the challenge map") continue } @@ -65,7 +64,6 @@ func syncOpenChallenges(ctx context.Context) { // challenge is not synced in db yet if errors.Is(err, gorm.ErrRecordNotFound) { - latestChallenge, err := GetLastChallengeEntity(tx) if err != nil { @@ -75,7 +73,7 @@ func syncOpenChallenges(ctx context.Context) { } } - isFirstChallengeInDatabase := len(challengeObj.PrevChallengeID) == 0 || latestChallenge == nil + isFirstChallengeInDatabase := challengeObj.PrevChallengeID == "" || latestChallenge == nil isNextChallengeOnChain := latestChallenge == nil || latestChallenge.ChallengeID == challengeObj.PrevChallengeID if isFirstChallengeInDatabase || isNextChallengeOnChain { @@ -90,13 +88,11 @@ func syncOpenChallenges(ctx context.Context) { } else { logging.Logger.Error("[challenge]Challenge chain is not valid") } - } db.Commit() tx.Done() } } - } } @@ -136,7 +132,6 @@ func processAccepted(ctx context.Context) { logging.Logger.Error("[challenge]db: ", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) return } - }(ctx, openchallenge) } swg.Wait() diff --git a/code/go/0chain.net/blobbercore/challenge/entity.go b/code/go/0chain.net/blobbercore/challenge/entity.go index bea780f62..94c5b0435 100644 --- a/code/go/0chain.net/blobbercore/challenge/entity.go +++ b/code/go/0chain.net/blobbercore/challenge/entity.go @@ -123,9 +123,7 @@ func (cr *ChallengeEntity) Save(ctx context.Context) error { if err != nil { return err } - //j, _ := json.Marshal(&cr.ObjectPathString) - // Logger.Info("Object path", zap.Any("objectpath", string(j))) - // Logger.Info("Object path object", zap.Any("object_path", cr.ObjectPath)) + db := datastore.GetStore().GetTransaction(ctx) err = db.Save(cr).Error return err diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index e3f9a5b0f..8db4deefa 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -28,7 +28,6 @@ type ChallengeResponse struct { } func (cr *ChallengeEntity) SubmitChallengeToBC(ctx context.Context) (*transaction.Transaction, error) { - txn, err := transaction.NewTransactionEntity() if err != nil { return nil, err @@ -100,7 +99,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { if rootRef.NumBlocks > 0 { r := rand.New(rand.NewSource(cr.RandomNumber)) blockNum = r.Int63n(rootRef.NumBlocks) - blockNum = blockNum + 1 + blockNum++ cr.BlockNum = blockNum } else { err = common.NewError("allocation_is_blank", "Got a challenge for a blank allocation") @@ -254,7 +253,6 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { } func (cr *ChallengeEntity) CommitChallenge(ctx context.Context, verifyOnly bool) error { - if len(cr.LastCommitTxnIDs) > 0 { for _, lastTxn := range cr.LastCommitTxnIDs { Logger.Info("[challenge]commit: Verifying the transaction : " + lastTxn) diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index 10e6ec5c9..ecf7773f1 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -26,7 +26,6 @@ func startCommitProcessed(ctx context.Context) { } func startProcessAccepted(ctx context.Context) { - for { select { case <-ctx.Done(): @@ -39,7 +38,6 @@ func startProcessAccepted(ctx context.Context) { // startSyncOpen func startSyncOpen(ctx context.Context) { - for { select { case <-ctx.Done(): diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 13b2401be..c574c44eb 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/viper" ) -//SetupDefaultConfig - setup the default config options that can be overridden via the config file +// SetupDefaultConfig - setup the default config options that can be overridden via the config file func SetupDefaultConfig() { viper.SetDefault("logging.level", "info") viper.SetDefault("contentref_cleaner.frequency", 30) @@ -156,7 +156,6 @@ func Geolocation() GeolocationConfig { if g.Latitude > 90.00 || g.Latitude < -90.00 || g.Longitude > 180.00 || g.Longitude < -180.00 { panic("Fatal error in config file") - } return g } diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index e1855d415..ab399015f 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -441,7 +441,7 @@ func WriteFileGRPCToHTTP(req *blobbergrpc.UploadFileRequest) (*http.Request, err "Invalid parameters. Error parsing the meta data for upload."+err.Error()) } - r, err := http.NewRequest(req.Method, "", nil) + r, err := http.NewRequest(req.Method, "", http.NoBody) if err != nil { return nil, err } @@ -477,7 +477,6 @@ func WriteFileGRPCToHTTP(req *blobbergrpc.UploadFileRequest) (*http.Request, err return nil, err } r.Header.Set("Content-Type", writer.FormDataContentType()) - } return r, nil diff --git a/code/go/0chain.net/blobbercore/convert/response_creator.go b/code/go/0chain.net/blobbercore/convert/response_creator.go index f228536fb..34cbd1b80 100644 --- a/code/go/0chain.net/blobbercore/convert/response_creator.go +++ b/code/go/0chain.net/blobbercore/convert/response_creator.go @@ -2,6 +2,7 @@ package convert import ( "encoding/json" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" @@ -28,7 +29,8 @@ func GetFileMetaDataResponseCreator(httpResp interface{}) *blobbergrpc.GetFileMe var resp blobbergrpc.GetFileMetaDataResponse collaborators, _ := r["collaborators"].([]reference.Collaborator) - for _, c := range collaborators { + for i := 0; i < len(collaborators); i++ { + c := collaborators[i] resp.Collaborators = append(resp.Collaborators, CollaboratorToGRPCCollaborator(&c)) } @@ -170,7 +172,8 @@ func CollaboratorResponseCreator(r interface{}) *blobbergrpc.CollaboratorRespons } collabs, _ := r.([]reference.Collaborator) - for _, c := range collabs { + for i := 0; i < len(collabs); i++ { + c := collabs[i] resp.Collaborators = append(resp.Collaborators, CollaboratorToGRPCCollaborator(&c)) } diff --git a/code/go/0chain.net/blobbercore/datastore/mock_store.go b/code/go/0chain.net/blobbercore/datastore/mock_store.go index 1b0de35eb..330fe986e 100644 --- a/code/go/0chain.net/blobbercore/datastore/mock_store.go +++ b/code/go/0chain.net/blobbercore/datastore/mock_store.go @@ -26,7 +26,6 @@ func MocketTheStore(t *testing.T, logging bool) { gdb, err := gorm.Open(dialect, new(gorm.Config)) require.NoError(t, err) - //setDB(gdb) instance = &postgresStore{ db: gdb, diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go index 248695263..3e8f9be7a 100644 --- a/code/go/0chain.net/blobbercore/datastore/mocket.go +++ b/code/go/0chain.net/blobbercore/datastore/mocket.go @@ -33,7 +33,6 @@ type Mocket struct { } func (store *Mocket) Open() error { - mocket.Catcher.Reset() mocket.Catcher.Register() mocket.Catcher.Logging = store.logging @@ -62,7 +61,6 @@ func (store *Mocket) Open() error { func (store *Mocket) Close() { if store.db != nil { - if db, _ := store.db.DB(); db != nil { db.Close() } diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go index f13fbb365..1dc998035 100644 --- a/code/go/0chain.net/blobbercore/datastore/sqlmock.go +++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go @@ -31,7 +31,6 @@ type Sqlmock struct { } func (store *Sqlmock) Open() error { - db, mock, err := sqlmock.New() if err != nil { return err @@ -57,7 +56,6 @@ func (store *Sqlmock) Open() error { func (store *Sqlmock) Close() { if store.db != nil { - if db, _ := store.db.DB(); db != nil { db.Close() } diff --git a/code/go/0chain.net/blobbercore/filestore/chunk_writer.go b/code/go/0chain.net/blobbercore/filestore/chunk_writer.go index e1e027f03..091466114 100644 --- a/code/go/0chain.net/blobbercore/filestore/chunk_writer.go +++ b/code/go/0chain.net/blobbercore/filestore/chunk_writer.go @@ -7,7 +7,7 @@ import ( "os" ) -//ChunkWriter implements a chunk write that will append content to the file +// ChunkWriter implements a chunk write that will append content to the file type ChunkWriter struct { file string writer *os.File @@ -16,7 +16,7 @@ type ChunkWriter struct { size int64 } -//NewChunkWriter create a ChunkWriter +// NewChunkWriter create a ChunkWriter func NewChunkWriter(file string) (*ChunkWriter, error) { w := &ChunkWriter{ file: file, diff --git a/code/go/0chain.net/blobbercore/filestore/chunk_writer_test.go b/code/go/0chain.net/blobbercore/filestore/chunk_writer_test.go index 716dc3189..54a28cffc 100644 --- a/code/go/0chain.net/blobbercore/filestore/chunk_writer_test.go +++ b/code/go/0chain.net/blobbercore/filestore/chunk_writer_test.go @@ -15,7 +15,6 @@ import ( ) func TestWrite(t *testing.T) { - fileName := filepath.Join(os.TempDir(), "testwrite_"+strconv.FormatInt(time.Now().Unix(), 10)) content := "this is full content" @@ -46,7 +45,6 @@ func TestWrite(t *testing.T) { } func TestWriteChunk(t *testing.T) { - chunk1 := "this is 1st chunked" tempFile, err := ioutil.TempFile("", "") @@ -55,7 +53,7 @@ func TestWriteChunk(t *testing.T) { require.Error(t, err, "failed to create tempfile") return } - offset, err := tempFile.Write([]byte(chunk1)) + offset, err := tempFile.WriteString(chunk1) if err != nil { require.Error(t, err, "failed to write first chunk to tempfile") return diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index 33d0b9f3c..f68812872 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -55,7 +55,7 @@ type IFileBlockGetter interface { type FileBlockGetter struct { } -func (FileBlockGetter) GetFileBlock(fs *FileFSStore, allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { +func (FileBlockGetter) GetFileBlock(fs *FileFSStore, allocationID string, fileData *FileInputData, blockNum, numBlocks int64) ([]byte, error) { allocation, err := fs.SetupAllocation(allocationID, true) if err != nil { return nil, common.NewError("invalid_allocation", "Invalid allocation. "+err.Error()) @@ -95,8 +95,8 @@ func (FileBlockGetter) GetFileBlock(fs *FileFSStore, allocationID string, fileDa if blockNum > maxBlockNum || blockNum < 1 { return nil, common.NewError("invalid_block_number", "Invalid block number") } - buffer := make([]byte, int64(fileData.ChunkSize)*numBlocks) - n, err := file.ReadAt(buffer, ((blockNum - 1) * int64(fileData.ChunkSize))) + buffer := make([]byte, fileData.ChunkSize*numBlocks) + n, err := file.ReadAt(buffer, ((blockNum - 1) * fileData.ChunkSize)) if err != nil && err != io.EOF { return nil, err } @@ -237,17 +237,16 @@ func (fs *FileFSStore) GetlDiskSizeUsed(allocationID string) (int64, error) { return size, err } -func GetFilePathFromHash(hash string) (string, string) { +func GetFilePathFromHash(h string) (string, string) { var dir bytes.Buffer - fmt.Fprintf(&dir, "%s", hash[0:3]) + fmt.Fprintf(&dir, "%s", h[0:3]) for i := 1; i < 3; i++ { - fmt.Fprintf(&dir, "%s%s", string(os.PathSeparator), hash[3*i:3*i+3]) + fmt.Fprintf(&dir, "%s%s", string(os.PathSeparator), h[3*i:3*i+3]) } - return dir.String(), hash[9:] + return dir.String(), h[9:] } func (fs *FileFSStore) generateTransactionPath(transID string) string { - var dir bytes.Buffer fmt.Fprintf(&dir, "%s%s", fs.RootDirectory, OSPathSeperator) for i := 0; i < 3; i++ { @@ -357,10 +356,8 @@ func (fs *FileFSStore) GetFileBlockForChallenge(allocationID string, fileData *F return returnBytes, fmt.GetMerkleTree(), nil } -func (fs *FileFSStore) GetFileBlock(allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { - +func (fs *FileFSStore) GetFileBlock(allocationID string, fileData *FileInputData, blockNum, numBlocks int64) ([]byte, error) { return fs.fileBlockGetter.GetFileBlock(fs, allocationID, fileData, blockNum, numBlocks) - } func (fs *FileFSStore) DeleteTempFile(allocationID string, fileData *FileInputData, connectionID string) error { @@ -430,7 +427,7 @@ func (fs *FileFSStore) CommitWrite(allocationID string, fileData *FileInputData, //return false, err } -func (fs *FileFSStore) DeleteFile(allocationID string, contentHash string) error { +func (fs *FileFSStore) DeleteFile(allocationID, contentHash string) error { allocation, err := fs.SetupAllocation(allocationID, true) if err != nil { return common.NewError("filestore_setup_error", "Error setting the fs store. "+err.Error()) @@ -454,9 +451,7 @@ func (fs *FileFSStore) DeleteDir(allocationID, dirPath, connectionID string) err return nil } -func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, - infile multipart.File, connectionID string) (*FileOutputData, error) { - +func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, infile multipart.File, connectionID string) (*FileOutputData, error) { if fileData.IsChunked { return fs.WriteChunk(allocationID, fileData, infile, connectionID) } @@ -486,7 +481,6 @@ func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, } fileSize := int64(0) for { - written, err := io.CopyN(dest, tReader, CHUNK_SIZE) if err != io.EOF && err != nil { @@ -526,9 +520,7 @@ func (fs *FileFSStore) WriteFile(allocationID string, fileData *FileInputData, } // WriteChunk append chunk to temp file -func (fs *FileFSStore) WriteChunk(allocationID string, fileData *FileInputData, - infile multipart.File, connectionID string) (*FileOutputData, error) { - +func (fs *FileFSStore) WriteChunk(allocationID string, fileData *FileInputData, infile multipart.File, connectionID string) (*FileOutputData, error) { allocation, err := fs.SetupAllocation(allocationID, false) if err != nil { return nil, common.NewError("filestore_setup_error", "Error setting the fs store. "+err.Error()) @@ -543,7 +535,7 @@ func (fs *FileFSStore) WriteChunk(allocationID string, fileData *FileInputData, fileRef := &FileOutputData{} - // the chunk has been rewitten. but it is lost when network is broken, and it is not save in db + // the chunk has been rewritten. but it is lost when network is broken, and it is not save in db if dest.size > fileData.UploadOffset { fileRef.ChunkUploaded = true } diff --git a/code/go/0chain.net/blobbercore/filestore/mock_store.go b/code/go/0chain.net/blobbercore/filestore/mock_store.go index 1b87ff6af..3eb356f76 100644 --- a/code/go/0chain.net/blobbercore/filestore/mock_store.go +++ b/code/go/0chain.net/blobbercore/filestore/mock_store.go @@ -19,7 +19,6 @@ var mockStore *MockStore func UseMock() { if mockStore == nil { mockStore = &MockStore{} - } fileStore = mockStore @@ -35,7 +34,6 @@ func (ms *MockStore) WriteFile(allocationID string, fileData *FileInputData, inf reader := io.TeeReader(infile, h) fileSize := int64(0) for { - written, err := io.CopyN(io.Discard, reader, fileData.ChunkSize) fileSize += written @@ -64,7 +62,7 @@ func (ms *MockStore) DeleteDir(allocationID, dirPath, connectionID string) error return nil } -func (ms *MockStore) GetFileBlock(allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { +func (ms *MockStore) GetFileBlock(allocationID string, fileData *FileInputData, blockNum, numBlocks int64) ([]byte, error) { return nil, constants.ErrNotImplemented } @@ -75,7 +73,7 @@ func (ms *MockStore) CommitWrite(allocationID string, fileData *FileInputData, c func (ms *MockStore) GetFileBlockForChallenge(allocationID string, fileData *FileInputData, blockoffset int) (json.RawMessage, util.MerkleTreeI, error) { return nil, nil, constants.ErrNotImplemented } -func (ms *MockStore) DeleteFile(allocationID string, contentHash string) error { +func (ms *MockStore) DeleteFile(allocationID, contentHash string) error { return nil } func (ms *MockStore) GetTotalDiskSizeUsed() (int64, error) { diff --git a/code/go/0chain.net/blobbercore/handler/calculate_hash_integration_test.go b/code/go/0chain.net/blobbercore/handler/calculate_hash_integration_test.go index 90285c35c..22126b88d 100644 --- a/code/go/0chain.net/blobbercore/handler/calculate_hash_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/calculate_hash_integration_test.go @@ -2,9 +2,10 @@ package handler import ( "context" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "testing" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "google.golang.org/grpc/metadata" diff --git a/code/go/0chain.net/blobbercore/handler/collaborator_integration_test.go b/code/go/0chain.net/blobbercore/handler/collaborator_integration_test.go index d06d60dcb..7f6c3eaa0 100644 --- a/code/go/0chain.net/blobbercore/handler/collaborator_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/collaborator_integration_test.go @@ -3,12 +3,13 @@ package handler import ( "context" "encoding/hex" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "net/http" "strconv" "testing" "time" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" diff --git a/code/go/0chain.net/blobbercore/handler/commit_meta_txn_integration_test.go b/code/go/0chain.net/blobbercore/handler/commit_meta_txn_integration_test.go index 23ddd72f9..380d1fb40 100644 --- a/code/go/0chain.net/blobbercore/handler/commit_meta_txn_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/commit_meta_txn_integration_test.go @@ -3,11 +3,12 @@ package handler import ( "context" "encoding/hex" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "strconv" "testing" "time" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" diff --git a/code/go/0chain.net/blobbercore/handler/copy_object_integration_test.go b/code/go/0chain.net/blobbercore/handler/copy_object_integration_test.go index befc6f1f6..2f3e93185 100644 --- a/code/go/0chain.net/blobbercore/handler/copy_object_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/copy_object_integration_test.go @@ -3,9 +3,10 @@ package handler import ( "context" "encoding/hex" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "testing" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "google.golang.org/grpc/metadata" diff --git a/code/go/0chain.net/blobbercore/handler/file_command.go b/code/go/0chain.net/blobbercore/handler/file_command.go index 9601b5248..08ed1e076 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command.go +++ b/code/go/0chain.net/blobbercore/handler/file_command.go @@ -19,7 +19,7 @@ type FileCommand interface { // ProcessThumbnail flush thumbnail file to FileStorage if it has. ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error - // UpdateChange update AllocationChangeProcessor. It will be president in db for commiting transcation + // UpdateChange update AllocationChangeProcessor. It will be president in db for committing transcation UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_add.go b/code/go/0chain.net/blobbercore/handler/file_command_add.go index 3907b6309..94c69a251 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_add.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_add.go @@ -52,7 +52,6 @@ func (cmd *AddFileCommand) IsAuthorized(ctx context.Context, req *http.Request, cmd.fileChanger = fileChanger return nil - } // ProcessContent flush file to FileStorage @@ -121,11 +120,9 @@ func (cmd *AddFileCommand) ProcessContent(ctx context.Context, req *http.Request // ProcessThumbnail flush thumbnail file to FileStorage if it has. func (cmd *AddFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { - thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") if thumbHeader != nil { - defer thumbfile.Close() thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} @@ -147,40 +144,42 @@ func (cmd *AddFileCommand) ProcessThumbnail(ctx context.Context, req *http.Reque func (cmd *AddFileCommand) reloadChange(connectionObj *allocation.AllocationChangeCollector) { for _, c := range connectionObj.Changes { - if c.Operation == constants.FileOperationInsert { + if c.Operation != constants.FileOperationInsert { + continue + } - dbChangeProcessor := &allocation.AddFileChanger{} + dbChangeProcessor := &allocation.AddFileChanger{} - err := dbChangeProcessor.Unmarshal(c.Input) - if err != nil { - logging.Logger.Error("reloadChange", zap.Error(err)) - } + err := dbChangeProcessor.Unmarshal(c.Input) + if err != nil { + logging.Logger.Error("reloadChange", zap.Error(err)) + } - cmd.fileChanger.Size = dbChangeProcessor.Size - cmd.fileChanger.ThumbnailFilename = dbChangeProcessor.ThumbnailFilename - cmd.fileChanger.ThumbnailSize = dbChangeProcessor.ThumbnailSize - cmd.fileChanger.ThumbnailHash = dbChangeProcessor.Hash + cmd.fileChanger.Size = dbChangeProcessor.Size + cmd.fileChanger.ThumbnailFilename = dbChangeProcessor.ThumbnailFilename + cmd.fileChanger.ThumbnailSize = dbChangeProcessor.ThumbnailSize + cmd.fileChanger.ThumbnailHash = dbChangeProcessor.Hash - return - } + return } } // UpdateChange replace AddFileChange in db func (cmd *AddFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { for _, c := range connectionObj.Changes { - if c.Operation == constants.FileOperationInsert { - c.Size = connectionObj.Size - c.Input, _ = cmd.fileChanger.Marshal() - - //c.ModelWithTS.UpdatedAt = time.Now() - err := connectionObj.Save(ctx) - if err != nil { - return err - } + if c.Operation != constants.FileOperationInsert { + continue + } + c.Size = connectionObj.Size + c.Input, _ = cmd.fileChanger.Marshal() - return c.Save(ctx) + //c.ModelWithTS.UpdatedAt = time.Now() + err := connectionObj.Save(ctx) + if err != nil { + return err } + + return c.Save(ctx) } //NOT FOUND diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index f33ae7014..1cdb288dd 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -25,7 +25,7 @@ func (cmd *FileCommandDelete) IsAuthorized(ctx context.Context, req *http.Reques } path := req.FormValue("path") - if len(path) == 0 { + if path == "" { return common.NewError("invalid_parameters", "Invalid path") } cmd.exisitingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, path) @@ -46,7 +46,6 @@ func (cmd *FileCommandDelete) UpdateChange(ctx context.Context, connectionObj *a // ProcessContent flush file to FileStorage func (cmd *FileCommandDelete) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { - deleteSize := cmd.exisitingFileRef.Size cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionObj.ConnectionID, @@ -67,7 +66,6 @@ func (cmd *FileCommandDelete) ProcessContent(ctx context.Context, req *http.Requ connectionObj.Size += cmd.allocationChange.Size return result, nil - } // ProcessThumbnail no thumbnail should be processed for delete. A deffered delete command has been added on ProcessContent diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index 5149a5919..69b1fbd46 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -60,7 +60,6 @@ func (cmd *UpdateFileCommand) IsAuthorized(ctx context.Context, req *http.Reques // ProcessContent flush file to FileStorage func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { - result := blobberhttp.UploadResult{} result.Filename = cmd.fileChanger.Filename @@ -125,16 +124,13 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ } return result, nil - } // ProcessThumbnail flush thumbnail file to FileStorage if it has. func (cmd *UpdateFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { - thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile") if thumbHeader != nil { - defer thumbfile.Close() thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} @@ -151,45 +147,47 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(ctx context.Context, req *http.Re } return nil - } func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationChangeCollector) { for _, c := range connectionObj.Changes { - if c.Operation == constants.FileOperationUpdate { - - dbFileChanger := &allocation.UpdateFileChanger{} + if c.Operation != constants.FileOperationUpdate { + continue + } - err := dbFileChanger.Unmarshal(c.Input) - if err != nil { - logging.Logger.Error("reloadChange", zap.Error(err)) - } + dbFileChanger := &allocation.UpdateFileChanger{} - // reload uploaded size from db, it was chunk size from client - cmd.fileChanger.Size = dbFileChanger.Size - cmd.fileChanger.ThumbnailFilename = dbFileChanger.ThumbnailFilename - cmd.fileChanger.ThumbnailSize = dbFileChanger.ThumbnailSize - cmd.fileChanger.ThumbnailHash = dbFileChanger.Hash - return + err := dbFileChanger.Unmarshal(c.Input) + if err != nil { + logging.Logger.Error("reloadChange", zap.Error(err)) } + + // reload uploaded size from db, it was chunk size from client + cmd.fileChanger.Size = dbFileChanger.Size + cmd.fileChanger.ThumbnailFilename = dbFileChanger.ThumbnailFilename + cmd.fileChanger.ThumbnailSize = dbFileChanger.ThumbnailSize + cmd.fileChanger.ThumbnailHash = dbFileChanger.Hash + return } } // UpdateChange add UpdateFileChanger in db func (cmd *UpdateFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error { for _, c := range connectionObj.Changes { - if c.Operation == constants.FileOperationUpdate { - c.Size = connectionObj.Size - c.Input, _ = cmd.fileChanger.Marshal() + if c.Operation != constants.FileOperationUpdate { + continue + } - //c.ModelWithTS.UpdatedAt = time.Now() - err := connectionObj.Save(ctx) - if err != nil { - return err - } + c.Size = connectionObj.Size + c.Input, _ = cmd.fileChanger.Marshal() - return c.Save(ctx) + //c.ModelWithTS.UpdatedAt = time.Now() + err := connectionObj.Save(ctx) + if err != nil { + return err } + + return c.Save(ctx) } //NOT FOUND diff --git a/code/go/0chain.net/blobbercore/handler/getallocation_integration_test.go b/code/go/0chain.net/blobbercore/handler/getallocation_integration_test.go index 171323bc1..4e44a5b9f 100644 --- a/code/go/0chain.net/blobbercore/handler/getallocation_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/getallocation_integration_test.go @@ -8,7 +8,6 @@ import ( ) func TestGetAllocation_IntegrationTest(t *testing.T) { - bClient, tdController := setupHandlerIntegrationTests(t) err := tdController.ClearDatabase() diff --git a/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go b/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go index f5dfef241..fe80e0a51 100644 --- a/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/getfilemetadata_integration_test.go @@ -2,11 +2,12 @@ package handler import ( "context" + "testing" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "google.golang.org/grpc/metadata" - "testing" ) func TestGetFileMetaData_IntegrationTest(t *testing.T) { diff --git a/code/go/0chain.net/blobbercore/handler/getfilestats_integration_test.go b/code/go/0chain.net/blobbercore/handler/getfilestats_integration_test.go index 47d97f439..032c2d74c 100644 --- a/code/go/0chain.net/blobbercore/handler/getfilestats_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/getfilestats_integration_test.go @@ -82,5 +82,4 @@ func TestGetFilestats_IntegrationTest(t *testing.T) { t.Fatal("unexpected file name from GetFileStats rpc") } } - } diff --git a/code/go/0chain.net/blobbercore/handler/getobjecttree_integration_test.go b/code/go/0chain.net/blobbercore/handler/getobjecttree_integration_test.go index bb596a4a5..e4ea3d759 100644 --- a/code/go/0chain.net/blobbercore/handler/getobjecttree_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/getobjecttree_integration_test.go @@ -80,5 +80,4 @@ func TestBlobberGRPCService_GetObjectTree(t *testing.T) { t.Fatal("unexpected root name from GetObject") } } - } diff --git a/code/go/0chain.net/blobbercore/handler/getreferencepath_integration_test.go b/code/go/0chain.net/blobbercore/handler/getreferencepath_integration_test.go index f5344c43e..f21632fbc 100644 --- a/code/go/0chain.net/blobbercore/handler/getreferencepath_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/getreferencepath_integration_test.go @@ -11,7 +11,6 @@ import ( ) func TestBlobberGRPCService_GetReferencePath(t *testing.T) { - bClient, tdController := setupHandlerIntegrationTests(t) allocationTx := randString(32) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler.go b/code/go/0chain.net/blobbercore/handler/grpc_handler.go index 67989eeb6..9fdff0d54 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler.go @@ -2,10 +2,11 @@ package handler import ( "context" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "net/http" "strings" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" ) @@ -18,7 +19,7 @@ func newGRPCBlobberService() *blobberGRPCService { } func (b *blobberGRPCService) GetAllocation(ctx context.Context, request *blobbergrpc.GetAllocationRequest) (*blobbergrpc.GetAllocationResponse, error) { - r, err := http.NewRequest("GET", "", nil) + r, err := http.NewRequest("GET", "", http.NoBody) if err != nil { return nil, err } @@ -34,7 +35,7 @@ func (b *blobberGRPCService) GetAllocation(ctx context.Context, request *blobber } func (b *blobberGRPCService) GetFileMetaData(ctx context.Context, req *blobbergrpc.GetFileMetaDataRequest) (*blobbergrpc.GetFileMetaDataResponse, error) { - r, err := http.NewRequest("POST", "", nil) + r, err := http.NewRequest("POST", "", http.NoBody) if err != nil { return nil, err } @@ -54,7 +55,7 @@ func (b *blobberGRPCService) GetFileMetaData(ctx context.Context, req *blobbergr } func (b *blobberGRPCService) GetFileStats(ctx context.Context, req *blobbergrpc.GetFileStatsRequest) (*blobbergrpc.GetFileStatsResponse, error) { - r, err := http.NewRequest("POST", "", nil) + r, err := http.NewRequest("POST", "", http.NoBody) if err != nil { return nil, err } @@ -73,7 +74,7 @@ func (b *blobberGRPCService) GetFileStats(ctx context.Context, req *blobbergrpc. } func (b *blobberGRPCService) ListEntities(ctx context.Context, req *blobbergrpc.ListEntitiesRequest) (*blobbergrpc.ListEntitiesResponse, error) { - r, err := http.NewRequest("", "", nil) + r, err := http.NewRequest("", "", http.NoBody) if err != nil { return nil, err } @@ -93,7 +94,7 @@ func (b *blobberGRPCService) ListEntities(ctx context.Context, req *blobbergrpc. } func (b *blobberGRPCService) GetObjectPath(ctx context.Context, req *blobbergrpc.GetObjectPathRequest) (*blobbergrpc.GetObjectPathResponse, error) { - r, err := http.NewRequest("", "", nil) + r, err := http.NewRequest("", "", http.NoBody) if err != nil { return nil, err } @@ -112,7 +113,7 @@ func (b *blobberGRPCService) GetObjectPath(ctx context.Context, req *blobbergrpc } func (b *blobberGRPCService) GetReferencePath(ctx context.Context, req *blobbergrpc.GetReferencePathRequest) (*blobbergrpc.GetReferencePathResponse, error) { - r, err := http.NewRequest("", "", nil) + r, err := http.NewRequest("", "", http.NoBody) if err != nil { return nil, err } @@ -131,7 +132,7 @@ func (b *blobberGRPCService) GetReferencePath(ctx context.Context, req *blobberg } func (b *blobberGRPCService) GetObjectTree(ctx context.Context, req *blobbergrpc.GetObjectTreeRequest) (*blobbergrpc.GetObjectTreeResponse, error) { - r, err := http.NewRequest("", "", nil) + r, err := http.NewRequest("", "", http.NoBody) if err != nil { return nil, err } @@ -149,7 +150,7 @@ func (b *blobberGRPCService) GetObjectTree(ctx context.Context, req *blobbergrpc } func (b *blobberGRPCService) CalculateHash(ctx context.Context, req *blobbergrpc.CalculateHashRequest) (*blobbergrpc.CalculateHashResponse, error) { - r, err := http.NewRequest("POST", "", nil) + r, err := http.NewRequest("POST", "", http.NoBody) if err != nil { return nil, err } @@ -168,7 +169,7 @@ func (b *blobberGRPCService) CalculateHash(ctx context.Context, req *blobbergrpc } func (b *blobberGRPCService) CommitMetaTxn(ctx context.Context, req *blobbergrpc.CommitMetaTxnRequest) (*blobbergrpc.CommitMetaTxnResponse, error) { - r, err := http.NewRequest("POST", "", nil) + r, err := http.NewRequest("POST", "", http.NoBody) if err != nil { return nil, err } @@ -189,7 +190,7 @@ func (b *blobberGRPCService) CommitMetaTxn(ctx context.Context, req *blobbergrpc } func (b *blobberGRPCService) Collaborator(ctx context.Context, req *blobbergrpc.CollaboratorRequest) (*blobbergrpc.CollaboratorResponse, error) { - r, err := http.NewRequest(strings.ToUpper(req.Method), "", nil) + r, err := http.NewRequest(strings.ToUpper(req.Method), "", http.NoBody) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go index 128b82296..d2efa87b9 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go @@ -123,7 +123,6 @@ func Test_GetAllocation(t *testing.T) { sqlmock.NewRows([]string{"id", "allocation_id"}). AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), ) - }, args: args{ allocationR: &blobbergrpc.GetAllocationRequest{ @@ -135,7 +134,7 @@ func Test_GetAllocation(t *testing.T) { wantAlloc: convert.AllocationToGRPCAllocation(alloc), }, { - name: "Commiting_Transaction_ERR", + name: "Committing_Transaction_ERR", mockSetup: func(mock sqlmock.Sqlmock) { mock.ExpectBegin() @@ -154,7 +153,6 @@ func Test_GetAllocation(t *testing.T) { sqlmock.NewRows([]string{"id", "allocation_id"}). AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), ) - }, args: args{ allocationR: &blobbergrpc.GetAllocationRequest{ @@ -185,7 +183,6 @@ func Test_GetAllocation(t *testing.T) { sqlmock.NewRows([]string{"id", "allocation_id"}). AddRow(expiredAlloc.Terms[0].ID, expiredAlloc.Terms[0].AllocationID), ) - }, args: args{ allocationR: &blobbergrpc.GetAllocationRequest{ diff --git a/code/go/0chain.net/blobbercore/handler/grpc_middleware.go b/code/go/0chain.net/blobbercore/handler/grpc_middleware.go index f1e73d530..60e321e4a 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_middleware.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_middleware.go @@ -49,7 +49,7 @@ func unaryDatabaseTransactionInjector() grpc.UnaryServerInterceptor { func unaryTimeoutInterceptor() grpc.UnaryServerInterceptor { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - deadline := time.Now().Add(time.Duration(TIMEOUT_SECONDS * time.Second)) + deadline := time.Now().Add(TIMEOUT_SECONDS * time.Second) ctx, canceler := context.WithDeadline(ctx, deadline) defer canceler() diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 17ab203f7..e1c90b8f7 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -40,7 +40,6 @@ func GetMetaDataStore() datastore.Store { /*SetupHandlers sets up the necessary API end points */ func SetupHandlers(r *mux.Router) { - r.Use(useRecovery, useCors, common.UseUserRateLimit) //object operations @@ -90,9 +89,7 @@ func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF } func WithConnection(handler common.JSONResponderF) common.JSONResponderF { - return func(ctx context.Context, r *http.Request) ( - resp interface{}, err error) { - + return func(ctx context.Context, r *http.Request) (resp interface{}, err error) { ctx = GetMetaDataStore().CreateTransaction(ctx) resp, err = handler(ctx, r) @@ -428,7 +425,7 @@ func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { if clientID != allocationObj.OwnerID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - err = reference.DeleteShareInfo(ctx, reference.ShareInfo{ + err = reference.DeleteShareInfo(ctx, &reference.ShareInfo{ ClientID: refereeClientID, FilePathHash: filePathHash, }) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index bc7b8f1e5..3149122e9 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -51,7 +51,7 @@ func (MockFileBlockGetter) GetFileBlock( fileData *filestore.FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { - return []byte(mockFileBlock), nil + return mockFileBlock, nil } func setMockFileBlock(data []byte) { @@ -128,8 +128,8 @@ func setup(t *testing.T) { } } -func setupHandlers() (*mux.Router, map[string]string) { - router := mux.NewRouter() +func setupHandlers() (router *mux.Router, opMap map[string]string) { + router = mux.NewRouter() opPath := "/v1/file/objectpath/{allocation}" opName := "Object_Path" @@ -264,7 +264,7 @@ func isEndpointAllowGetReq(name string) bool { } } -func GetAuthTicketForEncryptedFile(allocationID string, remotePath string, fileHash string, clientID string, encPublicKey string) (string, error) { +func GetAuthTicketForEncryptedFile(allocationID, remotePath, fileHash, clientID, encPublicKey string) (string, error) { at := &marker.AuthTicket{} at.AllocationID = allocationID at.OwnerID = client.GetClientID() @@ -371,7 +371,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { if !isEndpointAllowGetReq(name) { method = http.MethodPost } - r, err := http.NewRequest(method, url.String(), nil) + r, err := http.NewRequest(method, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -400,7 +400,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { if !isEndpointAllowGetReq(name) { method = http.MethodPost } - r, err := http.NewRequest(method, url.String(), nil) + r, err := http.NewRequest(method, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -441,7 +441,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), nil) + r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -501,7 +501,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), nil) + r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -561,7 +561,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), nil) + r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -632,7 +632,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), nil) + r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -693,7 +693,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("collab_id", "collab id") url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), nil) + r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -762,7 +762,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("connection_id", connectionID) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), nil) + r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -842,7 +842,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("dest", "dest") url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), nil) + r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -933,7 +933,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("attributes", string(attrBytes)) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), nil) + r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) if err != nil { t.Fatal(err) } @@ -1361,7 +1361,6 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). WithArgs(true, "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", filePathHash). WillReturnResult(sqlmock.NewResult(0, 1)) - }, wantCode: http.StatusOK, wantBody: "{\"message\":\"Path successfully removed from allocation\",\"status\":204}\n", @@ -1441,7 +1440,6 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). WithArgs(true, "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", filePathHash). WillReturnResult(sqlmock.NewResult(0, 0)) - }, wantCode: http.StatusOK, wantBody: "{\"message\":\"Path not found\",\"status\":404}\n", @@ -1526,7 +1524,6 @@ func TestHandlers_Requiring_Signature(t *testing.T) { filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, filePathHash).WillReturnError(gorm.ErrRecordNotFound) - }, wantCode: http.StatusBadRequest, wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: invalid file path: record not found\"}\n\n", @@ -1827,7 +1824,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } header := make([]byte, 2*1024) - copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) }, @@ -1966,7 +1963,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } header := make([]byte, 2*1024) - copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) }, @@ -2113,7 +2110,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } header := make([]byte, 2*1024) - copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) }, @@ -2260,7 +2257,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } header := make([]byte, 2*1024) - copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) }, @@ -2305,7 +2302,6 @@ func TestHandlers_Requiring_Signature(t *testing.T) { sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "/"), ) - }, wantCode: http.StatusBadRequest, wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: cannot verify auth ticket: invalid_parameters: Auth ticket is not valid for the resource being requested\"}\n\n", diff --git a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go index 0b92ca7cb..e8f61578e 100644 --- a/code/go/0chain.net/blobbercore/handler/helper_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/helper_integration_test.go @@ -33,7 +33,6 @@ const RetryAttempts = 8 const RetryTimeout = 3 func randString(n int) string { - const hexLetters = "abcdef0123456789" var sb strings.Builder @@ -502,9 +501,8 @@ VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','ro return nil } -func GeneratePubPrivateKey(t *testing.T) (string, string, zcncrypto.SignatureScheme) { - - signScheme := zcncrypto.NewSignatureScheme("bls0chain") +func GeneratePubPrivateKey(t *testing.T) (pubKey, privateKey string, signScheme zcncrypto.SignatureScheme) { + signScheme = zcncrypto.NewSignatureScheme("bls0chain") wallet, err := signScheme.GenerateKeys() if err != nil { t.Fatal(err) @@ -516,7 +514,6 @@ func GeneratePubPrivateKey(t *testing.T) (string, string, zcncrypto.SignatureSch } func setupIntegrationTestConfig(t *testing.T) { - pwd, err := os.Getwd() if err != nil { t.Fatal(err) diff --git a/code/go/0chain.net/blobbercore/handler/listentities_integration_test.go b/code/go/0chain.net/blobbercore/handler/listentities_integration_test.go index 3df112dd4..68adb0a79 100644 --- a/code/go/0chain.net/blobbercore/handler/listentities_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/listentities_integration_test.go @@ -2,9 +2,9 @@ package handler import ( "context" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "testing" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" @@ -86,5 +86,4 @@ func TestBlobberGRPCService_ListEntities(t *testing.T) { t.Fatal("unexpected path from ListEntities rpc") } } - } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go index dcb069c77..518b19d31 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go @@ -2,13 +2,14 @@ package handler import ( "context" + "net/http" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" - "net/http" ) func (b *blobberGRPCService) UpdateObjectAttributes(ctx context.Context, req *blobbergrpc.UpdateObjectAttributesRequest) (*blobbergrpc.UpdateObjectAttributesResponse, error) { - r, err := http.NewRequest("POST", "", nil) + r, err := http.NewRequest("POST", "", http.NoBody) if err != nil { return nil, err } @@ -29,7 +30,7 @@ func (b *blobberGRPCService) UpdateObjectAttributes(ctx context.Context, req *bl } func (b *blobberGRPCService) CopyObject(ctx context.Context, req *blobbergrpc.CopyObjectRequest) (*blobbergrpc.CopyObjectResponse, error) { - r, err := http.NewRequest("POST", "", nil) + r, err := http.NewRequest("POST", "", http.NoBody) if err != nil { return nil, err } @@ -50,7 +51,7 @@ func (b *blobberGRPCService) CopyObject(ctx context.Context, req *blobbergrpc.Co } func (b *blobberGRPCService) RenameObject(ctx context.Context, req *blobbergrpc.RenameObjectRequest) (*blobbergrpc.RenameObjectResponse, error) { - r, err := http.NewRequest("POST", "", nil) + r, err := http.NewRequest("POST", "", http.NoBody) if err != nil { return nil, err } @@ -87,7 +88,6 @@ func (b *blobberGRPCService) DownloadFile(ctx context.Context, req *blobbergrpc. } func (b *blobberGRPCService) UploadFile(ctx context.Context, req *blobbergrpc.UploadFileRequest) (*blobbergrpc.UploadFileResponse, error) { - r, err := convert.WriteFileGRPCToHTTP(req) if err != nil { return nil, err diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index d64b5c72b..32d48d1f8 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -39,9 +39,7 @@ import ( "go.uber.org/zap" ) -func readPreRedeem(ctx context.Context, alloc *allocation.Allocation, - numBlocks, pendNumBlocks int64, payerID string) (err error) { - +func readPreRedeem(ctx context.Context, alloc *allocation.Allocation, numBlocks, pendNumBlocks int64, payerID string) (err error) { if numBlocks == 0 { return } @@ -105,9 +103,7 @@ func readPreRedeem(ctx context.Context, alloc *allocation.Allocation, return } -func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, - writeMarker *writemarker.WriteMarker, payerID string) (err error) { - +func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMarker *writemarker.WriteMarker, payerID string) (err error) { // check out read pool tokens if read_price > 0 var ( db = datastore.GetStore().GetTransaction(ctx) @@ -178,11 +174,7 @@ func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, return } -func (fsh *StorageHandler) DownloadFile( - ctx context.Context, - r *http.Request, -) (resp interface{}, err error) { - +func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (resp interface{}, err error) { // get client and allocation ids var ( clientID = ctx.Value(constants.ContextKeyClient).(string) @@ -192,7 +184,7 @@ func (fsh *StorageHandler) DownloadFile( ) // check client - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("download_file", "invalid client") } @@ -204,7 +196,7 @@ func (fsh *StorageHandler) DownloadFile( } // get and parse file params - if err = r.ParseMultipartForm(FormFileParseMaxMemory); nil != err { + if err = r.ParseMultipartForm(FormFileParseMaxMemory); err != nil { Logger.Info("download_file - request_parse_error", zap.Error(err)) return nil, common.NewErrorf("download_file", "request_parse_error: %v", err) @@ -216,7 +208,7 @@ func (fsh *StorageHandler) DownloadFile( } var blockNumStr = r.FormValue("block_num") - if len(blockNumStr) == 0 { + if blockNumStr == "" { return nil, common.NewError("download_file", "no block number") } @@ -227,7 +219,7 @@ func (fsh *StorageHandler) DownloadFile( } var numBlocksStr = r.FormValue("num_blocks") - if len(numBlocksStr) == 0 { + if numBlocksStr == "" { numBlocksStr = "1" } @@ -333,10 +325,8 @@ func (fsh *StorageHandler) DownloadFile( if fileAttrs, err := fileref.GetAttributes(); err != nil { return nil, common.NewErrorf("download_file", "error getting file attributes: %v", err) - } else { - if fileAttrs.WhoPaysForReads == common.WhoPays3rdParty { - payerID = clientID - } + } else if fileAttrs.WhoPaysForReads == common.WhoPays3rdParty { + payerID = clientID } } @@ -361,9 +351,7 @@ func (fsh *StorageHandler) DownloadFile( } } - if latestRM != nil && - latestRM.ReadCounter+(numBlocks) != readMarker.ReadCounter { - + if latestRM != nil && latestRM.ReadCounter+(numBlocks) != readMarker.ReadCounter { var response = &blobberhttp.DownloadResponse{ Success: false, LatestRM: latestRM, @@ -435,7 +423,6 @@ func (fsh *StorageHandler) DownloadFile( } if len(fileref.EncryptedKey) > 0 && authToken != nil { - // should not happen, just in case if shareInfo == nil { return nil, errors.New("client does not have permission to download the file. share does not exist") @@ -493,10 +480,10 @@ func (fsh *StorageHandler) DownloadFile( } func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*blobberhttp.CommitResult, error) { - if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use POST instead") } + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) clientID := ctx.Value(constants.ContextKeyClient).(string) clientKey := ctx.Value(constants.ContextKeyClientKey).(string) @@ -514,7 +501,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b allocationID := allocationObj.ID connectionID := r.FormValue("connection_id") - if len(connectionID) == 0 { + if connectionID == "" { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } @@ -534,21 +521,23 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b var isCollaborator bool for _, change := range connectionObj.Changes { - if change.Operation == constants.FileOperationUpdate { - updateFileChange := new(allocation.UpdateFileChanger) - if err := updateFileChange.Unmarshal(change.Input); err != nil { - return nil, err - } - fileRef, err := reference.GetReference(ctx, allocationID, updateFileChange.Path) - if err != nil { - return nil, err - } - isCollaborator = reference.IsACollaborator(ctx, fileRef.ID, clientID) - break + if change.Operation != constants.FileOperationUpdate { + continue + } + + updateFileChange := new(allocation.UpdateFileChanger) + if err := updateFileChange.Unmarshal(change.Input); err != nil { + return nil, err + } + fileRef, err := reference.GetReference(ctx, allocationID, updateFileChange.Path) + if err != nil { + return nil, err } + isCollaborator = reference.IsACollaborator(ctx, fileRef.ID, clientID) + break } - if len(clientID) == 0 || len(clientKey) == 0 { + if clientID == "" || clientKey == "" { return nil, common.NewError("invalid_params", "Please provide clientID and clientKey") } @@ -556,7 +545,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - if err = r.ParseMultipartForm(FormFileParseMaxMemory); nil != err { + if err = r.ParseMultipartForm(FormFileParseMaxMemory); err != nil { Logger.Info("Error Parsing the request", zap.Any("error", err)) return nil, common.NewError("request_parse_error", err.Error()) } @@ -577,7 +566,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b var result blobberhttp.CommitResult var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { + if allocationObj.AllocationRoot == "" { latestWM = nil } else { latestWM, err = writemarker.GetWriteMarkerEntity(ctx, @@ -607,7 +596,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b clientIDForWriteRedeem = allocationObj.OwnerID } - if err = writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil { + if err := writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil { return nil, err } @@ -689,12 +678,12 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i return nil, common.NewError("invalid_signature", "Invalid signature") } - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Invalid client") } new_name := r.FormValue("new_name") - if len(new_name) == 0 { + if new_name == "" { return nil, common.NewError("invalid_parameters", "Invalid name") } @@ -703,12 +692,12 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i return nil, err } - if len(clientID) == 0 || allocationObj.OwnerID != clientID { + if clientID == "" || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } connectionID := r.FormValue("connection_id") - if len(connectionID) == 0 { + if connectionID == "" { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } @@ -752,9 +741,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i return result, nil } -func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, - r *http.Request) (resp interface{}, err error) { - +func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, r *http.Request) (resp interface{}, err error) { if r.Method != http.MethodPost { return nil, common.NewError("update_object_attributes", "Invalid method used. Use POST instead") @@ -886,12 +873,12 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int allocationID := allocationObj.ID - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Invalid client") } destPath := r.FormValue("dest") - if len(destPath) == 0 { + if destPath == "" { return nil, common.NewError("invalid_parameters", "Invalid destination for operation") } @@ -900,12 +887,12 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int return nil, err } - if len(clientID) == 0 || allocationObj.OwnerID != clientID { + if clientID == "" || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } connectionID := r.FormValue("connection_id") - if len(connectionID) == 0 { + if connectionID == "" { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } @@ -956,7 +943,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, connectionObj *allocation.AllocationChangeCollector) (*blobberhttp.UploadResult, error) { path := r.FormValue("path") - if len(path) == 0 { + if path == "" { return nil, common.NewError("invalid_parameters", "Invalid path") } @@ -1004,12 +991,12 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo allocationID := allocationObj.ID - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") } dirPath := r.FormValue("dir_path") - if len(dirPath) == 0 { + if dirPath == "" { return nil, common.NewError("invalid_parameters", "Invalid dir path passed") } @@ -1023,7 +1010,7 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo } connectionID := r.FormValue("connection_id") - if len(connectionID) == 0 { + if connectionID == "" { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } @@ -1067,7 +1054,6 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo //WriteFile stores the file into the blobber files system from the HTTP request func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blobberhttp.UploadResult, error) { - if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use multi-part form POST / PUT / DELETE / PATCH instead") } @@ -1100,7 +1086,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("immutable_allocation", "Cannot write to an immutable allocation") } - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") } @@ -1110,7 +1096,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo } connectionID := r.FormValue("connection_id") - if len(connectionID) == 0 { + if connectionID == "" { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go index 2020ebaa9..4763c426d 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go @@ -16,7 +16,6 @@ import ( ) func BenchmarkUploadFileWithDisk(b *testing.B) { - KB := 1024 MB := 1024 * KB //GB := 1024 * MB @@ -55,7 +54,6 @@ func BenchmarkUploadFileWithDisk(b *testing.B) { for _, bm := range benchmarks { b.Run(bm.Name, func(b *testing.B) { - fileName := strings.Replace(bm.Name, " ", "_", -1) + ".txt" chunkBytes := mock.GenerateRandomBytes(bm.ChunkSize) fileMeta := &sdk.FileMeta{ @@ -98,15 +96,12 @@ func BenchmarkUploadFileWithDisk(b *testing.B) { b.Fatal(err) return } - } }) } - } func BenchmarkUploadFileWithNoDisk(b *testing.B) { - KB := 1024 MB := 1024 * KB //GB := 1024 * MB @@ -146,7 +141,6 @@ func BenchmarkUploadFileWithNoDisk(b *testing.B) { for _, bm := range benchmarks { b.Run(bm.Name, func(b *testing.B) { - fileName := strings.Replace(bm.Name, " ", "_", -1) + ".txt" chunkBytes := mock.GenerateRandomBytes(bm.ChunkSize) fileMeta := &sdk.FileMeta{ @@ -180,7 +174,6 @@ func BenchmarkUploadFileWithNoDisk(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - ctx := GetMetaDataStore().CreateTransaction(context.TODO()) ctx = mock.SetupHandlerContext(ctx, req, allocationID) @@ -190,9 +183,7 @@ func BenchmarkUploadFileWithNoDisk(b *testing.B) { b.Fatal(err) return } - } }) } - } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index 82a6b4e02..f4dc23b8d 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -589,7 +589,6 @@ func TestDownloadFile(t *testing.T) { require.EqualValues(t, test.want.errMsg, err.Error()) return } - }, ) } diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go index 73e4cbe6c..7e45ec694 100644 --- a/code/go/0chain.net/blobbercore/handler/protocol.go +++ b/code/go/0chain.net/blobbercore/handler/protocol.go @@ -29,7 +29,7 @@ type WalletCallback struct { err string } -func (wb *WalletCallback) OnWalletCreateComplete(status int, wallet string, err string) { +func (wb *WalletCallback) OnWalletCreateComplete(status int, wallet, err string) { wb.err = err wb.wg.Done() } @@ -94,7 +94,6 @@ func getStorageNode() (*transaction.StorageNode, error) { // Add or update blobber on blockchain func BlobberAdd(ctx context.Context) (string, error) { - // initialize storage node (ie blobber) txn, err := transaction.NewTransactionEntity() if err != nil { @@ -131,7 +130,6 @@ func BlobberAdd(ctx context.Context) (string, error) { var ErrBlobberHasRemoved = errors.New("blobber has removed") func TransactionVerify(txnHash string) (t *transaction.Transaction, err error) { - for i := 0; i < util.MAX_RETRIES; i++ { time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second) if t, err = transaction.VerifyTransaction(txnHash, chain.GetServerChain()); err == nil { diff --git a/code/go/0chain.net/blobbercore/handler/renameobject_integration_test.go b/code/go/0chain.net/blobbercore/handler/renameobject_integration_test.go index ff053741a..96b530efb 100644 --- a/code/go/0chain.net/blobbercore/handler/renameobject_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/renameobject_integration_test.go @@ -3,9 +3,10 @@ package handler import ( "context" "encoding/hex" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "testing" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "google.golang.org/grpc/metadata" diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 5bae8d1a0..3c2b8c912 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -36,10 +36,8 @@ const ( type StorageHandler struct{} // verifyAllocation try to get allocation from postgres.if it doesn't exists, get it from sharders, and insert it into postgres. -func (fsh *StorageHandler) verifyAllocation(ctx context.Context, tx string, - readonly bool) (alloc *allocation.Allocation, err error) { - - if len(tx) == 0 { +func (fsh *StorageHandler) verifyAllocation(ctx context.Context, tx string, readonly bool) (alloc *allocation.Allocation, err error) { + if tx == "" { return nil, common.NewError("verify_allocation", "invalid allocation id") } @@ -69,7 +67,7 @@ func (fsh *StorageHandler) convertGormError(err error) error { } func (fsh *StorageHandler) verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string) (bool, error) { - if len(authTokenString) == 0 { + if authTokenString == "" { return false, common.NewError("invalid_parameters", "Auth ticket required if data read by anyone other than owner.") } authToken := &readmarker.AuthTicket{} @@ -135,7 +133,7 @@ func (fsh *StorageHandler) GetAllocationUpdateTicket(ctx context.Context, r *htt return allocationObj, nil } -func (fsh *StorageHandler) checkIfFileAlreadyExists(ctx context.Context, allocationID string, path string) *reference.Ref { +func (fsh *StorageHandler) checkIfFileAlreadyExists(ctx context.Context, allocationID, path string) *reference.Ref { fileReference, err := reference.GetReference(ctx, allocationID, path) if err != nil { return nil @@ -156,7 +154,7 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in allocationID := alloc.ID clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -234,7 +232,7 @@ func (fsh *StorageHandler) AddCommitMetaTxn(ctx context.Context, r *http.Request allocationID := allocationObj.ID clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -267,7 +265,7 @@ func (fsh *StorageHandler) AddCommitMetaTxn(ctx context.Context, r *http.Request } txnID := r.FormValue("txn_id") - if len(txnID) == 0 { + if txnID == "" { return nil, common.NewError("invalid_parameter", "TxnID not present in the params") } @@ -317,7 +315,7 @@ func (fsh *StorageHandler) AddCollaborator(ctx context.Context, r *http.Request) } collabClientID := r.FormValue("collab_id") - if len(collabClientID) == 0 { + if collabClientID == "" { return nil, common.NewError("invalid_parameter", "collab_id not present in the params") } @@ -327,7 +325,7 @@ func (fsh *StorageHandler) AddCollaborator(ctx context.Context, r *http.Request) switch r.Method { case http.MethodPost: - if len(clientID) == 0 || clientID != allocationObj.OwnerID { + if clientID == "" || clientID != allocationObj.OwnerID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -351,7 +349,7 @@ func (fsh *StorageHandler) AddCollaborator(ctx context.Context, r *http.Request) return collaborators, nil case http.MethodDelete: - if len(clientID) == 0 || clientID != allocationObj.OwnerID { + if clientID == "" || clientID != allocationObj.OwnerID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -386,7 +384,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i } clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 || allocationObj.OwnerID != clientID { + if clientID == "" || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -414,7 +412,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i } var statsMap map[string]interface{} statsBytes, _ := json.Marshal(fileStats) - if err = json.Unmarshal(statsBytes, &statsMap); err != nil { + if err := json.Unmarshal(statsBytes, &statsMap); err != nil { return nil, err } for k, v := range statsMap { @@ -424,7 +422,6 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i } func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*blobberhttp.ListResult, error) { - clientID := ctx.Value(constants.ContextKeyClient).(string) allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) @@ -433,7 +430,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } allocationID := allocationObj.ID - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -506,7 +503,6 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* result.Meta["collaborators"] = []reference.Collaborator{} } result.Meta["collaborators"] = append(result.Meta["collaborators"].([]reference.Collaborator), collaborators...) - } return &result, nil @@ -551,7 +547,7 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request } clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 { + if clientID == "" { errCh <- common.NewError("invalid_operation", "Please pass clientID in the header") return } @@ -588,7 +584,7 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request } var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { + if allocationObj.AllocationRoot == "" { latestWM = nil } else { latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) @@ -624,16 +620,16 @@ func (fsh *StorageHandler) GetObjectPath(ctx context.Context, r *http.Request) ( } clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 || allocationObj.OwnerID != clientID { + if clientID == "" || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } path := r.FormValue("path") - if len(path) == 0 { + if path == "" { return nil, common.NewError("invalid_parameters", "Invalid path") } blockNumStr := r.FormValue("block_num") - if len(blockNumStr) == 0 { + if blockNumStr == "" { return nil, common.NewError("invalid_parameters", "Invalid path") } @@ -648,7 +644,7 @@ func (fsh *StorageHandler) GetObjectPath(ctx context.Context, r *http.Request) ( } var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { + if allocationObj.AllocationRoot == "" { latestWM = nil } else { latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) @@ -683,11 +679,11 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( } clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 || allocationObj.OwnerID != clientID { + if clientID == "" || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } path := r.FormValue("path") - if len(path) == 0 { + if path == "" { return nil, common.NewError("invalid_parameters", "Invalid path") } @@ -715,7 +711,7 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( } var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { + if allocationObj.AllocationRoot == "" { latestWM = nil } else { latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) @@ -743,7 +739,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb } clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 { + if clientID == "" { return nil, common.NewError("invalid_operation", "Client id is required") } @@ -829,7 +825,6 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb } default: return nil, common.NewError("incomplete_request", "path, pathHash or authTicket is required") - } path = pathRef.Path @@ -865,7 +860,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb fileType := r.FormValue("fileType") levelStr := r.FormValue("level") var level int - if len(levelStr) != 0 { + if levelStr != "" { level, err = strconv.Atoi(levelStr) if err != nil { return nil, common.NewError("invalid_parameters", err.Error()) @@ -899,7 +894,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb return nil, err } var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { + if allocationObj.AllocationRoot == "" { latestWM = nil } else { latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) @@ -933,7 +928,7 @@ func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) ( allocationID := allocationObj.ID clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 || allocationObj.OwnerID != clientID { + if clientID == "" || allocationObj.OwnerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -957,19 +952,19 @@ func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) ( } // verifySignatureFromRequest verifies signature passed as common.ClientSignatureHeader header. -func verifySignatureFromRequest(allocation, sign, pbK string) (bool, error) { +func verifySignatureFromRequest(alloc, sign, pbK string) (bool, error) { sign = encryption.MiraclToHerumiSig(sign) if len(sign) < 64 { return false, nil } - hash := encryption.Hash(allocation) + hash := encryption.Hash(alloc) return encryption.Verify(pbK, sign, hash) } // pathsFromReq retrieves paths value from request which can be represented as single "path" value or "paths" values, -// marshalled to json. +// marshaled to json. func pathsFromReq(r *http.Request) ([]string, error) { var ( pathsStr = r.FormValue("paths") @@ -977,8 +972,8 @@ func pathsFromReq(r *http.Request) ([]string, error) { paths = make([]string, 0) ) - if len(pathsStr) == 0 { - if len(path) == 0 { + if pathsStr == "" { + if path == "" { return nil, common.NewError("invalid_parameters", "Invalid path") } @@ -997,8 +992,8 @@ func pathHashFromReq(r *http.Request, allocationID string) (string, error) { pathHash = r.FormValue("path_hash") path = r.FormValue("path") ) - if len(pathHash) == 0 { - if len(path) == 0 { + if pathHash == "" { + if path == "" { return "", common.NewError("invalid_parameters", "Invalid path") } pathHash = reference.GetReferenceLookup(allocationID, path) @@ -1007,13 +1002,12 @@ func pathHashFromReq(r *http.Request, allocationID string) (string, error) { return pathHash, nil } -func getPathHash(r *http.Request, allocationID string) (string, string, error) { - var ( - pathHash = r.FormValue("path_hash") - path = r.FormValue("path") - ) - if len(pathHash) == 0 { - if len(path) == 0 { +func getPathHash(r *http.Request, allocationID string) (pathHash, path string, err error) { + pathHash = r.FormValue("path_hash") + path = r.FormValue("path") + + if pathHash == "" { + if path == "" { return "", "", common.NewError("invalid_parameters", "Invalid path") } pathHash = reference.GetReferenceLookup(allocationID, path) diff --git a/code/go/0chain.net/blobbercore/handler/updateattributes_integration_test.go b/code/go/0chain.net/blobbercore/handler/updateattributes_integration_test.go index 0050b8504..a6522c286 100644 --- a/code/go/0chain.net/blobbercore/handler/updateattributes_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/updateattributes_integration_test.go @@ -4,9 +4,10 @@ import ( "context" "encoding/hex" "encoding/json" - blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" "testing" + blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 3aa705f1d..cb161e864 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -66,8 +66,8 @@ func cleanupTempFiles(ctx context.Context) { var openConnectionsToDelete []allocation.AllocationChangeCollector db.Table((&allocation.AllocationChangeCollector{}).TableName()).Where("updated_at < ? AND status IN (?,?)", then, allocation.NewConnection, allocation.InProgressConnection).Preload("Changes").Find(&openConnectionsToDelete) - for _, connection := range openConnectionsToDelete { + for _, connection := range openConnectionsToDelete { Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ConnectionID)) connection.ComputeProperties() nctx := datastore.GetStore().CreateTransaction(ctx) @@ -103,7 +103,7 @@ func startCleanupTempFiles(ctx context.Context) { } } -func moveColdDataToCloud(ctx context.Context, coldStorageMinFileSize int64, limit int64) { +func moveColdDataToCloud(ctx context.Context, coldStorageMinFileSize, limit int64) { defer func() { if r := recover(); r != nil { Logger.Error("[recover] moveColdDataToCloud", zap.Any("err", r)) @@ -153,7 +153,7 @@ func moveColdDataToCloud(ctx context.Context, coldStorageMinFileSize int64, limi moveFileToCloud(ctx, fileRef) } } - offset = offset + limit + offset += limit } db.Commit() rctx.Done() @@ -171,7 +171,6 @@ func startMoveColdDataToCloud(ctx context.Context) { return case <-ticker.C: if !iterInprogress { - moveColdDataToCloud(ctx, coldStorageMinFileSize, limit) iterInprogress = false stats.LastMinioScan = time.Now() diff --git a/code/go/0chain.net/blobbercore/handler/zcncore.go b/code/go/0chain.net/blobbercore/handler/zcncore.go index dbdbbf09a..1a50293bb 100644 --- a/code/go/0chain.net/blobbercore/handler/zcncore.go +++ b/code/go/0chain.net/blobbercore/handler/zcncore.go @@ -25,7 +25,7 @@ func (zcn *ZCNStatus) OnBalanceAvailable(status int, value int64, info string) { zcn.balance = value } -func (zcn *ZCNStatus) OnInfoAvailable(op int, status int, info string, err string) { +func (zcn *ZCNStatus) OnInfoAvailable(op, status int, info, err string) { defer zcn.wg.Done() if status == zcncore.StatusSuccess { zcn.success = true @@ -152,5 +152,4 @@ func Transfer(token float64, clientID string) error { return common.NewError("call_transfer_failed", "Failed to verify send transaction with statusBar success failed") } return nil - } diff --git a/code/go/0chain.net/blobbercore/mock/allocation.go b/code/go/0chain.net/blobbercore/mock/allocation.go index 5d1896cf4..5e797b5fc 100644 --- a/code/go/0chain.net/blobbercore/mock/allocation.go +++ b/code/go/0chain.net/blobbercore/mock/allocation.go @@ -1,7 +1,7 @@ package mock import ( - "github.com/selvatico/go-mocket" + gomocket "github.com/selvatico/go-mocket" ) func MockGetAllocationByID(allocationID string, allocation map[string]interface{}) { diff --git a/code/go/0chain.net/blobbercore/mock/ctx.go b/code/go/0chain.net/blobbercore/mock/ctx.go index 7573aa48f..b41344c2e 100644 --- a/code/go/0chain.net/blobbercore/mock/ctx.go +++ b/code/go/0chain.net/blobbercore/mock/ctx.go @@ -9,7 +9,6 @@ import ( ) func SetupHandlerContext(ctx context.Context, r *http.Request, allocation string) context.Context { - ctx = context.WithValue(ctx, constants.ContextKeyClient, r.Header.Get(common.ClientHeader)) ctx = context.WithValue(ctx, constants.ContextKeyClientKey, diff --git a/code/go/0chain.net/blobbercore/mock/init.go b/code/go/0chain.net/blobbercore/mock/init.go index 768e21bd9..2564584ad 100644 --- a/code/go/0chain.net/blobbercore/mock/init.go +++ b/code/go/0chain.net/blobbercore/mock/init.go @@ -18,7 +18,6 @@ const ( // ) func NewBlobberClient() *blobber.Blobber { - z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain") err := z.InitWallet(zboxWallet) if err != nil { diff --git a/code/go/0chain.net/blobbercore/readmarker/entity.go b/code/go/0chain.net/blobbercore/readmarker/entity.go index 8f3d516ec..f0e6a20de 100644 --- a/code/go/0chain.net/blobbercore/readmarker/entity.go +++ b/code/go/0chain.net/blobbercore/readmarker/entity.go @@ -126,7 +126,6 @@ func SaveLatestReadMarker(ctx context.Context, rm *ReadMarker, isCreate bool) er // Sync read marker with 0chain to be sure its correct. func (rm *ReadMarkerEntity) Sync(ctx context.Context) (err error) { - var db = datastore.GetStore().GetTransaction(ctx) ////////////////////////////////////////////////////////////////////////////// @@ -166,11 +165,8 @@ func (rm *ReadMarkerEntity) Sync(ctx context.Context) (err error) { return } -// UpdateStatus updates read marker status and all related on successful -// redeeming. -func (rm *ReadMarkerEntity) UpdateStatus(ctx context.Context, - rps []*allocation.ReadPool, txOutput, redeemTxn string) (err error) { - +// UpdateStatus updates read marker status and all related on successful redeeming. +func (rm *ReadMarkerEntity) UpdateStatus(ctx context.Context, rps []*allocation.ReadPool, txOutput, redeemTxn string) (err error) { var redeems []allocation.ReadPoolRedeem if err = json.Unmarshal([]byte(txOutput), &redeems); err != nil { Logger.Error("update read redeeming status: can't decode transaction"+ @@ -191,7 +187,7 @@ func (rm *ReadMarkerEntity) UpdateStatus(ctx context.Context, return common.NewErrorf("rme_update_status", "marshaling latest RM: %v", err) } - if len(latestRMBytes) == 0 || string(latestRMBytes) == "" { + if len(latestRMBytes) == 0 { latestRMBytes = []byte("{}") } rmUpdates["latest_redeemed_rm"] = latestRMBytes diff --git a/code/go/0chain.net/blobbercore/readmarker/protocol.go b/code/go/0chain.net/blobbercore/readmarker/protocol.go index 87795e6c1..0aec3fd3f 100644 --- a/code/go/0chain.net/blobbercore/readmarker/protocol.go +++ b/code/go/0chain.net/blobbercore/readmarker/protocol.go @@ -36,12 +36,12 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All } clientPublicKey := ctx.Value(constants.ContextKeyClientKey).(string) - if len(clientPublicKey) == 0 || clientPublicKey != rm.LatestRM.ClientPublicKey { + if clientPublicKey == "" || clientPublicKey != rm.LatestRM.ClientPublicKey { return common.NewError("read_marker_validation_failed", "Could not get the public key of the client") } clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 || clientID != rm.LatestRM.ClientID { + if clientID == "" || clientID != rm.LatestRM.ClientID { return common.NewError("read_marker_validation_failed", "Read Marker clientID does not match request clientID") } currentTS := common.Now() @@ -70,7 +70,6 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All // returns difference between latest redeemed read marker and current one // (till not redeemed). func (rme *ReadMarkerEntity) PendNumBlocks() (pendNumBlocks int64, err error) { - if !rme.RedeemRequired { return // (0, nil), everything is already redeemed } @@ -94,13 +93,11 @@ func (rme *ReadMarkerEntity) PendNumBlocks() (pendNumBlocks int64, err error) { pendNumBlocks = rme.LatestRM.ReadCounter - prev.ReadCounter return - } // getNumBlocks to redeem (difference between the previous RM and the // current one) func (rme *ReadMarkerEntity) getNumBlocks() (numBlocks int64, err error) { - if rme.LatestRM == nil { return 0, common.NewErrorf("rme_get_num_blocks", "missing latest read marker (nil)") @@ -126,10 +123,7 @@ func (rme *ReadMarkerEntity) getNumBlocks() (numBlocks int64, err error) { // a redeeming transaction regarding cache, pending reads (regardless since // pending reads is what we are going to redeem) and requesting 0chain to // refresh read pools -func (rme *ReadMarkerEntity) preRedeem(ctx context.Context, - alloc *allocation.Allocation, numBlocks int64) ( - rps []*allocation.ReadPool, err error) { - +func (rme *ReadMarkerEntity) preRedeem(ctx context.Context, alloc *allocation.Allocation, numBlocks int64) (rps []*allocation.ReadPool, err error) { // check out read pool tokens if read_price > 0 var ( db = datastore.GetStore().GetTransaction(ctx) @@ -198,9 +192,7 @@ func (rme *ReadMarkerEntity) preRedeem(ctx context.Context, } // RedeemReadMarker redeems the read marker. -func (rme *ReadMarkerEntity) RedeemReadMarker(ctx context.Context) ( - err error) { - +func (rme *ReadMarkerEntity) RedeemReadMarker(ctx context.Context) (err error) { if rme.LatestRM.Suspend == rme.LatestRM.ReadCounter { // suspended read marker, no tokens in related read pools // don't request 0chain to refresh the read pools; let user diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go index 665e442a9..29198b2d1 100644 --- a/code/go/0chain.net/blobbercore/readmarker/readmarker.go +++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go @@ -16,7 +16,6 @@ func redeemReadMarker(ctx context.Context) { if r := recover(); r != nil { logging.Logger.Error("[recover] redeemReadMarker", zap.Any("err", r)) } - }() if !iterInprogress { @@ -42,7 +41,7 @@ func redeemReadMarker(ctx context.Context) { db := datastore.GetStore().GetTransaction(redeemCtx) err = db.Commit().Error if err != nil { - logging.Logger.Error("Error commiting the readmarker redeem", zap.Error(err)) + logging.Logger.Error("Error committing the readmarker redeem", zap.Error(err)) } swg.Done() }(ctx, rmEntity) diff --git a/code/go/0chain.net/blobbercore/readmarker/worker.go b/code/go/0chain.net/blobbercore/readmarker/worker.go index 7f30f838a..4b77669e9 100644 --- a/code/go/0chain.net/blobbercore/readmarker/worker.go +++ b/code/go/0chain.net/blobbercore/readmarker/worker.go @@ -17,9 +17,7 @@ func SetupWorkers(ctx context.Context) { go startRedeemMarkers(ctx) } -func RedeemReadMarker(ctx context.Context, rmEntity *ReadMarkerEntity) ( - err error) { - +func RedeemReadMarker(ctx context.Context, rmEntity *ReadMarkerEntity) (err error) { Logger.Info("Redeeming the read marker", zap.Any("rm", rmEntity.LatestRM)) var params = make(map[string]string) @@ -38,13 +36,10 @@ func RedeemReadMarker(ctx context.Context, rmEntity *ReadMarkerEntity) ( if err != nil { Logger.Error("Error from sc rest api call", zap.Error(err)) return // error - } else if err = json.Unmarshal(latestRMBytes, &latestRM); err != nil { Logger.Error("Error from unmarshal of rm bytes", zap.Error(err)) return // error - } else if latestRM.ReadCounter > 0 && latestRM.ReadCounter >= rmEntity.LatestRM.ReadCounter { - Logger.Info("updating the local state to match the block chain") if err = SaveLatestReadMarker(ctx, &latestRM, false); err != nil { return // error @@ -87,5 +82,4 @@ func startRedeemMarkers(ctx context.Context) { redeemReadMarker(ctx) } } - } diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 2cae89491..28575d38d 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -28,13 +28,11 @@ func LoadObjectTree(ctx context.Context, allocationID, path string) (*Ref, error err := db.FindInBatches(&objects, 100, func(tx *gorm.DB, batch int) error { // batch processing found records for _, object := range objects { - obejctTreeNodes[object.ParentPath] = append(obejctTreeNodes[object.ParentPath], object) for _, child := range obejctTreeNodes[object.Path] { object.AddChild(child) } - } return nil @@ -57,16 +55,13 @@ func LoadObjectTree(ctx context.Context, allocationID, path string) (*Ref, error } return nil, common.NewError("invalid_ref_tree", "/ is missing or invalid") - } return nil, common.NewError("invalid_ref_tree", "/ is missing") - } // DeleteObject delete object from tree, and return tree root and deleted content hash list -func DeleteObject(ctx context.Context, allocationID string, path string) (*Ref, map[string]bool, error) { - +func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[string]bool, error) { rootRef, err := LoadObjectTree(ctx, allocationID, "/") if err != nil { return nil, nil, err @@ -126,5 +121,4 @@ func DeleteObject(ctx context.Context, allocationID string, path string) (*Ref, } } return nil, nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") - } diff --git a/code/go/0chain.net/blobbercore/reference/objectpath.go b/code/go/0chain.net/blobbercore/reference/objectpath.go index 84d449131..c36a79e72 100644 --- a/code/go/0chain.net/blobbercore/reference/objectpath.go +++ b/code/go/0chain.net/blobbercore/reference/objectpath.go @@ -19,7 +19,6 @@ type ObjectPath struct { // TODO needs to be refactored, current implementation can probably be heavily simplified func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*ObjectPath, error) { - rootRef, err := GetRefWithSortedChildren(ctx, allocationID, "/") if err != nil { return nil, common.NewError("invalid_dir_struct", "Allocation root corresponds to an invalid directory structure") @@ -58,9 +57,8 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O } curResult["list"] = list for idx, child := range curRef.Children { - if child.NumBlocks < remainingBlocks { - remainingBlocks = remainingBlocks - child.NumBlocks + remainingBlocks -= child.NumBlocks continue } if child.Type == FILE { @@ -69,7 +67,7 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O break } curRef, err = GetRefWithSortedChildren(ctx, allocationID, child.Path) - if err != nil || len(curRef.Hash) == 0 { + if err != nil || curRef.Hash == "" { return nil, common.NewError("failed_object_path", "Failed to get the object path") } curResult = list[idx] diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 078153fb1..80deec905 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -136,7 +136,7 @@ func (Ref) TableName() string { } // GetReferenceLookup hash(allocationID + ":" + path) -func GetReferenceLookup(allocationID string, path string) string { +func GetReferenceLookup(allocationID, path string) string { return encryption.Hash(allocationID + ":" + path) } @@ -174,7 +174,7 @@ func (r *Ref) SetAttributes(attr *Attributes) (err error) { } // Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child -func Mkdir(ctx context.Context, allocationID string, destpath string) (*Ref, error) { +func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { var dirRef *Ref db := datastore.GetStore().GetTransaction(ctx) // cleaning path to avoid edge case issues: append '/' prefix if not added and removing suffix '/' if added @@ -212,11 +212,10 @@ func Mkdir(ctx context.Context, allocationID string, destpath string) (*Ref, err } return dirRef, nil - } // GetReference get FileRef with allcationID and path from postgres -func GetReference(ctx context.Context, allocationID string, path string) (*Ref, error) { +func GetReference(ctx context.Context, allocationID, path string) (*Ref, error) { ref := &Ref{} db := datastore.GetStore().GetTransaction(ctx) err := db.Where(&Ref{AllocationID: allocationID, Path: path}).First(ref).Error @@ -226,7 +225,7 @@ func GetReference(ctx context.Context, allocationID string, path string) (*Ref, return nil, err } -func GetReferenceFromLookupHash(ctx context.Context, allocationID string, path_hash string) (*Ref, error) { +func GetReferenceFromLookupHash(ctx context.Context, allocationID, path_hash string) (*Ref, error) { ref := &Ref{} db := datastore.GetStore().GetTransaction(ctx) err := db.Where(&Ref{AllocationID: allocationID, LookupHash: path_hash}).First(ref).Error @@ -252,7 +251,7 @@ func GetSubDirsFromPath(p string) []string { return subDirs } -func GetRefWithChildren(ctx context.Context, allocationID string, path string) (*Ref, error) { +func GetRefWithChildren(ctx context.Context, allocationID, path string) (*Ref, error) { var refs []Ref db := datastore.GetStore().GetTransaction(ctx) db = db.Where(Ref{ParentPath: path, AllocationID: allocationID}).Or(Ref{Type: DIRECTORY, Path: path, AllocationID: allocationID}) @@ -277,7 +276,7 @@ func GetRefWithChildren(ctx context.Context, allocationID string, path string) ( return &refs[0], nil } -func GetRefWithSortedChildren(ctx context.Context, allocationID string, path string) (*Ref, error) { +func GetRefWithSortedChildren(ctx context.Context, allocationID, path string) (*Ref, error) { var refs []*Ref db := datastore.GetStore().GetTransaction(ctx) db = db.Where(Ref{ParentPath: path, AllocationID: allocationID}).Or(Ref{Type: DIRECTORY, Path: path, AllocationID: allocationID}) @@ -307,17 +306,19 @@ func (fr *Ref) GetFileHashData() string { fr.Attributes = datatypes.JSON("{}") } hashArray := make([]string, 0, 11) - hashArray = append(hashArray, fr.AllocationID) - hashArray = append(hashArray, fr.Type) - hashArray = append(hashArray, fr.Name) - hashArray = append(hashArray, fr.Path) - hashArray = append(hashArray, strconv.FormatInt(fr.Size, 10)) - hashArray = append(hashArray, fr.ContentHash) - hashArray = append(hashArray, fr.MerkleRoot) - hashArray = append(hashArray, strconv.FormatInt(fr.ActualFileSize, 10)) - hashArray = append(hashArray, fr.ActualFileHash) - hashArray = append(hashArray, string(fr.Attributes)) - hashArray = append(hashArray, strconv.FormatInt(fr.ChunkSize, 10)) + hashArray = append(hashArray, + fr.AllocationID, + fr.Type, + fr.Name, + fr.Path, + strconv.FormatInt(fr.Size, 10), + fr.ContentHash, + fr.MerkleRoot, + strconv.FormatInt(fr.ActualFileSize, 10), + fr.ActualFileHash, + string(fr.Attributes), + strconv.FormatInt(fr.ChunkSize, 10), + ) return strings.Join(hashArray, ":") } @@ -406,7 +407,7 @@ func (r *Ref) RemoveChild(idx int) { r.childrenLoaded = true } -func (r *Ref) UpdatePath(newPath string, parentPath string) { +func (r *Ref) UpdatePath(newPath, parentPath string) { r.Path = newPath r.ParentPath = parentPath r.PathLevel = len(GetSubDirsFromPath(r.Path)) + 1 //strings.Count(r.Path, "/") @@ -497,7 +498,6 @@ func GetListingFieldsMap(refEntity interface{}, tagName string) map[string]inter for k, v := range listMap { result[k] = v } - } } else { fieldValue := v.FieldByName(field.Name).Interface() @@ -506,7 +506,6 @@ func GetListingFieldsMap(refEntity interface{}, tagName string) map[string]inter } result[tag] = fieldValue } - } return result } diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 1e1580dd4..56eceb367 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -18,7 +18,7 @@ type ReferencePath struct { Ref *Ref } -func GetReferencePath(ctx context.Context, allocationID string, path string) (*Ref, error) { +func GetReferencePath(ctx context.Context, allocationID, path string) (*Ref, error) { return GetReferencePathFromPaths(ctx, allocationID, []string{path}) } @@ -64,7 +64,6 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths [ refMap := make(map[string]*Ref) refMap[rootRef.Path] = rootRef for i := 1; i < len(refs); i++ { - if _, ok := refMap[refs[i].ParentPath]; !ok { return nil, common.NewError("invalid_dir_tree", "DB has invalid tree.") } @@ -80,7 +79,7 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths [ return &refs[0], nil } -func PathExists(ctx context.Context, allocationID string, path string) (bool, error) { +func PathExists(ctx context.Context, allocationID, path string) (bool, error) { path = filepath.Clean(path) if path == "." || path == "/" { @@ -100,7 +99,7 @@ func PathExists(ctx context.Context, allocationID string, path string) (bool, er return true, nil } -func GetObjectTree(ctx context.Context, allocationID string, path string) (*Ref, error) { +func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) { path = filepath.Clean(path) var refs []Ref db := datastore.GetStore().GetTransaction(ctx) @@ -130,7 +129,7 @@ func GetObjectTree(ctx context.Context, allocationID string, path string) (*Ref, return &refs[0], nil } -//This function retrieves refrence_objects tables rows with pagination. Check for issue https://github.com/0chain/gosdk/issues/117 +//This function retrieves reference_objects tables rows with pagination. Check for issue https://github.com/0chain/gosdk/issues/117 //Might need to consider covering index for efficient search https://blog.crunchydata.com/blog/why-covering-indexes-are-incredibly-helpful //To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs //very easily and effectively; Same case for offsetDate. @@ -182,7 +181,6 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, refs = &pRefs if len(pRefs) > 0 { newOffsetPath = pRefs[len(pRefs)-1].Path - } totalPages = int(math.Ceil(float64(totalRows) / float64(pageLimit))) return @@ -282,7 +280,6 @@ func GetDeletedRefs(ctx context.Context, allocationID, updatedDate, offsetPath, }() go func() { - db2 = db2.Model(&Ref{}).Unscoped().Where("allocation_id = ?", allocationID) if updatedDate == "" { @@ -299,7 +296,6 @@ func GetDeletedRefs(ctx context.Context, allocationID, updatedDate, offsetPath, lastIdx := len(pRefs) - 1 newOffsetDate = pRefs[lastIdx].DeletedAt.Time.Format(dateLayOut) newOffsetPath = pRefs[lastIdx].Path - } refs = &pRefs totalPages = int(math.Ceil(float64(totalRows) / float64(pageLimit))) diff --git a/code/go/0chain.net/blobbercore/reference/shareinfo.go b/code/go/0chain.net/blobbercore/reference/shareinfo.go index 1caafae55..49e9cffea 100644 --- a/code/go/0chain.net/blobbercore/reference/shareinfo.go +++ b/code/go/0chain.net/blobbercore/reference/shareinfo.go @@ -28,7 +28,7 @@ func AddShareInfo(ctx context.Context, shareInfo ShareInfo) error { return db.Table(TableName()).Create(shareInfo).Error } -func DeleteShareInfo(ctx context.Context, shareInfo ShareInfo) error { +func DeleteShareInfo(ctx context.Context, shareInfo *ShareInfo) error { db := datastore.GetStore().GetTransaction(ctx) result := db.Table(TableName()). @@ -64,7 +64,7 @@ func UpdateShareInfo(ctx context.Context, shareInfo ShareInfo) error { Error } -func GetShareInfo(ctx context.Context, clientID string, filePathHash string) (*ShareInfo, error) { +func GetShareInfo(ctx context.Context, clientID, filePathHash string) (*ShareInfo, error) { db := datastore.GetStore().GetTransaction(ctx) shareInfo := &ShareInfo{} err := db.Table(TableName()). diff --git a/code/go/0chain.net/blobbercore/stats/allocationstats.go b/code/go/0chain.net/blobbercore/stats/allocationstats.go index a680d3245..55ddafaab 100644 --- a/code/go/0chain.net/blobbercore/stats/allocationstats.go +++ b/code/go/0chain.net/blobbercore/stats/allocationstats.go @@ -1,8 +1,9 @@ package stats import ( - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" ) // Timestamp that implements standard fmt.Stringer interface. diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index 831841c6f..0c30bafa1 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -230,7 +230,6 @@ func (bs *BlobberStats) loadFailedChallengeList(ctx context.Context) { } func (bs *BlobberStats) loadStats(ctx context.Context) { - const sel = ` COALESCE (SUM (reference_objects.size), 0) AS files_size, COALESCE (SUM (reference_objects.thumbnail_size), 0) AS thumbnails_size, @@ -267,7 +266,6 @@ func (bs *BlobberStats) loadStats(ctx context.Context) { } func (bs *BlobberStats) loadMinioStats(ctx context.Context) { - var ( db = datastore.GetStore().GetTransaction(ctx) row *sql.Row @@ -376,7 +374,6 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { } func (bs *BlobberStats) loadChallengeStats(ctx context.Context) { - var ( db = datastore.GetStore().GetTransaction(ctx) rows *sql.Rows @@ -429,7 +426,6 @@ func (bs *BlobberStats) loadChallengeStats(ctx context.Context) { zap.Error(err)) return } - } func (bs *BlobberStats) loadAllocationChallengeStats(ctx context.Context) { @@ -499,11 +495,9 @@ func (bs *BlobberStats) loadAllocationChallengeStats(ctx context.Context) { zap.Error(err)) return } - } func loadAllocationList(ctx context.Context) (interface{}, error) { - var ( allocations = make([]AllocationId, 0) db = datastore.GetStore().GetTransaction(ctx) @@ -550,9 +544,7 @@ type ReadMarkerEntity struct { RedeemRequired bool `gorm:"column:redeem_required"` } -func loadAllocReadMarkersStat(ctx context.Context, allocationID string) ( - rms *ReadMarkersStat, err error) { - +func loadAllocReadMarkersStat(ctx context.Context, allocationID string) (rms *ReadMarkersStat, err error) { var ( db = datastore.GetStore().GetTransaction(ctx) rme ReadMarkerEntity @@ -601,9 +593,7 @@ const ( Failed // 2 ) -func loadAllocWriteMarkerStat(ctx context.Context, allocationID string) ( - wms *WriteMarkersStat, err error) { - +func loadAllocWriteMarkerStat(ctx context.Context, allocationID string) (wms *WriteMarkersStat, err error) { var ( db = datastore.GetStore().GetTransaction(ctx) rows *sql.Rows diff --git a/code/go/0chain.net/blobbercore/stats/challengestats.go b/code/go/0chain.net/blobbercore/stats/challengestats.go index 18c1f926b..98b2127be 100644 --- a/code/go/0chain.net/blobbercore/stats/challengestats.go +++ b/code/go/0chain.net/blobbercore/stats/challengestats.go @@ -1,16 +1,13 @@ package stats import ( + "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "gorm.io/datatypes" - "time" ) -func (ChallengeEntity) TableName() string { - return "challenges" -} - type ChallengeEntity struct { ChallengeID string `json:"id" gorm:"column:challenge_id;primary_key"` PrevChallengeID string `json:"prev_id" gorm:"column:prev_challenge_id"` @@ -34,6 +31,9 @@ type ChallengeEntity struct { UpdatedAt time.Time `gorm:"updated_at"` } +func (ChallengeEntity) TableName() string { + return "challenges" +} func getAllFailedChallenges(offset, limit int) ([]ChallengeEntity, int, error) { db := datastore.GetStore().GetDB() crs := []ChallengeEntity{} diff --git a/code/go/0chain.net/blobbercore/stats/filestats.go b/code/go/0chain.net/blobbercore/stats/filestats.go index b6aebb826..eb17c8e15 100644 --- a/code/go/0chain.net/blobbercore/stats/filestats.go +++ b/code/go/0chain.net/blobbercore/stats/filestats.go @@ -2,6 +2,7 @@ package stats import ( "context" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "gorm.io/gorm" ) @@ -16,8 +17,6 @@ type FileStats struct { LastChallengeResponseTxn string `gorm:"column:last_challenge_txn" json:"last_challenge_txn"` WriteMarkerRedeemTxn string `gorm:"-" json:"write_marker_txn"` datastore.ModelWithTS - - //NumBlockWrites int64 `gorm:"column:num_of_block_writes" json:"num_of_block_writes"` } func (FileStats) TableName() string { diff --git a/code/go/0chain.net/blobbercore/stats/handler.go b/code/go/0chain.net/blobbercore/stats/handler.go index 3ca5e9912..f4e7e263a 100644 --- a/code/go/0chain.net/blobbercore/stats/handler.go +++ b/code/go/0chain.net/blobbercore/stats/handler.go @@ -3,14 +3,15 @@ package stats import ( "context" "fmt" + "html/template" + "net/http" + "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "go.uber.org/zap" - "html/template" - "net/http" - "time" ) func byteCountIEC(b int64) string { @@ -35,12 +36,8 @@ var funcMap = template.FuncMap{ "read_size": func(readCount int64) string { return byteCountIEC(readCount * readBlockSize) }, - "write_size": func(readCount int64) string { - return byteCountIEC(readCount) - }, - "byte_count_in_string": func(byteValue int64) string { - return byteCountIEC(byteValue) - }, + "write_size": byteCountIEC, + "byte_count_in_string": byteCountIEC, "time_in_string": func(timeValue time.Time) string { if timeValue.IsZero() { return "-" @@ -499,7 +496,7 @@ func GetStatsHandler(ctx context.Context, r *http.Request) (interface{}, error) defer db.Rollback() allocationID := ctx.Value(constants.ContextKeyAllocation).(string) bs := &BlobberStats{} - if len(allocationID) != 0 { + if allocationID != "" { // TODO: Get only the allocation info from DB bs.loadDetailedStats(ctx) for _, allocStat := range bs.AllocationStats { @@ -510,7 +507,7 @@ func GetStatsHandler(ctx context.Context, r *http.Request) (interface{}, error) return nil, common.NewError("allocation_stats_not_found", "Stats for allocation not found") } allocations := q.Get("allocations") - if len(allocations) != 0 { + if allocations != "" { return loadAllocationList(ctx) } bs.loadBasicStats(ctx) diff --git a/code/go/0chain.net/blobbercore/stats/pagination.go b/code/go/0chain.net/blobbercore/stats/pagination.go index fce3cadcf..a7a78954f 100644 --- a/code/go/0chain.net/blobbercore/stats/pagination.go +++ b/code/go/0chain.net/blobbercore/stats/pagination.go @@ -72,7 +72,7 @@ func GeneratePagination(page, limit, offset, totalItems int) *Pagination { p.TotalPages = totalItems / limit if totalItems%limit > 0 { - p.TotalPages = p.TotalPages + 1 + p.TotalPages++ } if p.TotalPages > p.CurrentPage { diff --git a/code/go/0chain.net/blobbercore/util/json.go b/code/go/0chain.net/blobbercore/util/json.go index 672eddfca..dbfce39ac 100644 --- a/code/go/0chain.net/blobbercore/util/json.go +++ b/code/go/0chain.net/blobbercore/util/json.go @@ -2,9 +2,10 @@ package util import ( "fmt" - "github.com/0chain/gosdk/zboxcore/fileref" "reflect" "strings" + + "github.com/0chain/gosdk/zboxcore/fileref" ) // Validate unmarshalled data with tag-based rules @@ -27,12 +28,12 @@ func UnmarshalValidation(v interface{}) error { return nil } -func GetParentPathHashes(allocationTx string, filePath string) []string { +func GetParentPathHashes(allocationTx, filePath string) []string { splitted := strings.Split(filePath, "/") pathHashes := []string{} for i := 0; i < len(splitted); i++ { - path := strings.Join(splitted[:len(splitted) - i], "/") + path := strings.Join(splitted[:len(splitted)-i], "/") if path == "" { path = "/" } diff --git a/code/go/0chain.net/blobbercore/writemarker/entity.go b/code/go/0chain.net/blobbercore/writemarker/entity.go index 52e4b10bb..586efef35 100644 --- a/code/go/0chain.net/blobbercore/writemarker/entity.go +++ b/code/go/0chain.net/blobbercore/writemarker/entity.go @@ -51,10 +51,7 @@ func (WriteMarkerEntity) TableName() string { return "write_markers" } -func (wm *WriteMarkerEntity) UpdateStatus(ctx context.Context, - status WriteMarkerStatus, statusMessage string, redeemTxn string) ( - err error) { - +func (wm *WriteMarkerEntity) UpdateStatus(ctx context.Context, status WriteMarkerStatus, statusMessage, redeemTxn string) (err error) { db := datastore.GetStore().GetTransaction(ctx) statusBytes, _ := json.Marshal(statusMessage) @@ -83,7 +80,7 @@ func (wm *WriteMarkerEntity) UpdateStatus(ctx context.Context, return // not committed or a deleting marker } - // work on pre-redeemd tokens and write-pools balances tracking + // work on pre-redeemed tokens and write-pools balances tracking var pend *allocation.Pending pend, err = allocation.GetPending(db, wm.WM.ClientID, wm.WM.AllocationID, @@ -109,7 +106,7 @@ func GetWriteMarkerEntity(ctx context.Context, allocation_root string) (*WriteMa return wm, nil } -func GetWriteMarkersInRange(ctx context.Context, allocationID string, startAllocationRoot string, endAllocationRoot string) ([]*WriteMarkerEntity, error) { +func GetWriteMarkersInRange(ctx context.Context, allocationID, startAllocationRoot, endAllocationRoot string) ([]*WriteMarkerEntity, error) { db := datastore.GetStore().GetTransaction(ctx) var seqRange []int64 err := db.Table((WriteMarkerEntity{}).TableName()). diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index cdad31027..5571cf005 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -50,12 +50,12 @@ func (wm *WriteMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.Al } clientPublicKey := ctx.Value(constants.ContextKeyClientKey).(string) - if len(clientPublicKey) == 0 { + if clientPublicKey == "" { return common.NewError("write_marker_validation_failed", "Could not get the public key of the client") } clientID := ctx.Value(constants.ContextKeyClient).(string) - if len(clientID) == 0 || clientID != wm.WM.ClientID || clientID != co.ClientID || co.ClientID != wm.WM.ClientID { + if clientID == "" || clientID != wm.WM.ClientID || clientID != co.ClientID || co.ClientID != wm.WM.ClientID { return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded") } @@ -73,7 +73,6 @@ func (wm *WriteMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.Al } func (wm *WriteMarkerEntity) RedeemMarker(ctx context.Context) error { - if len(wm.CloseTxnID) > 0 { t, err := transaction.VerifyTransaction(wm.CloseTxnID, chain.GetServerChain()) if err == nil { diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index 09fb734f4..151e78d2a 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -41,7 +41,7 @@ func RedeemMarkersForAllocation(ctx context.Context, allocationObj *allocation.A if wm.WM.PreviousAllocationRoot == allocationObj.LatestRedeemedWM && !startredeem { startredeem = true } - if startredeem || len(allocationObj.LatestRedeemedWM) == 0 { + if startredeem || allocationObj.LatestRedeemedWM == "" { err := wm.RedeemMarker(rctx) if err != nil { Logger.Error("Error redeeming the write marker.", zap.Any("wm", wm.WM.AllocationID), zap.Any("error", err)) @@ -61,7 +61,6 @@ func RedeemMarkersForAllocation(ctx context.Context, allocationObj *allocation.A Where("allocation_root = ? AND allocation_root = latest_redeemed_write_marker", allocationObj.AllocationRoot). Update("is_redeem_required", false) } - //Logger.Info("Returning from redeem", zap.Any("wm", latestWmEntity), zap.Any("allocation", allocationID)) return nil } @@ -79,5 +78,4 @@ func startRedeemWriteMarkers(ctx context.Context) { redeemWriteMarker(ctx) } } - } diff --git a/code/go/0chain.net/conductor/conductrpc/entity.go b/code/go/0chain.net/conductor/conductrpc/entity.go index fd983d2e4..ddf89fdf5 100644 --- a/code/go/0chain.net/conductor/conductrpc/entity.go +++ b/code/go/0chain.net/conductor/conductrpc/entity.go @@ -35,7 +35,6 @@ func (e *Entity) SetState(state *State) { // NewEntity creates RPC client for integration tests. func NewEntity(id string) (e *Entity) { - var ( client, err = newClient(viper.GetString("integration_tests.address")) interval = viper.GetDuration("integration_tests.lock_interval") diff --git a/code/go/0chain.net/conductor/conductrpc/state.go b/code/go/0chain.net/conductor/conductrpc/state.go index 7f723162f..5d3464e09 100644 --- a/code/go/0chain.net/conductor/conductrpc/state.go +++ b/code/go/0chain.net/conductor/conductrpc/state.go @@ -60,7 +60,6 @@ func (s *State) copy() (cp *State) { //nolint:unused,deadcode // might be used l cp = new(State) (*cp) = (*s) return - } func (s *State) send(poll chan *State) { //nolint:unused,deadcode // might be used later? diff --git a/code/go/0chain.net/core/build/info.go b/code/go/0chain.net/core/build/info.go index 6bbd32618..15cc3cc04 100644 --- a/code/go/0chain.net/core/build/info.go +++ b/code/go/0chain.net/core/build/info.go @@ -1,4 +1,4 @@ package build -//BuildTag - the git commit for the build +// BuildTag - the git commit for the build var BuildTag string diff --git a/code/go/0chain.net/core/cache/lfu.go b/code/go/0chain.net/core/cache/lfu.go index 5ac53d31f..b5c91a1cf 100644 --- a/code/go/0chain.net/core/cache/lfu.go +++ b/code/go/0chain.net/core/cache/lfu.go @@ -8,19 +8,19 @@ type LFU struct { Cache cache.Cache } -//NewLFUCache - create a new LFU cache object +// NewLFUCache - create a new LFU cache object func NewLFUCache(size int) *LFU { c := &LFU{} c.Cache = cache.NewLFU(size) return c } -//Add - add a given key and value +// Add - add a given key and value func (c *LFU) Add(key string, value interface{}) error { return c.Cache.Set(key, value) } -//Get - get the value associated with the key +// Get - get the value associated with the key func (c *LFU) Get(key string) (interface{}, error) { value, err := c.Cache.Get(key) if err != nil { diff --git a/code/go/0chain.net/core/cache/lru.go b/code/go/0chain.net/core/cache/lru.go index 31c650b3a..53dd1e630 100644 --- a/code/go/0chain.net/core/cache/lru.go +++ b/code/go/0chain.net/core/cache/lru.go @@ -4,24 +4,24 @@ import ( lru "github.com/koding/cache" ) -//LRU - LRU cache +// LRU - LRU cache type LRU struct { Cache lru.Cache } -//NewLRUCache - create a new LRU cache +// NewLRUCache - create a new LRU cache func NewLRUCache(size int) *LRU { c := &LRU{} c.Cache = lru.NewLRU(size) return c } -//Add - add a key and a value +// Add - add a key and a value func (c *LRU) Add(key string, value interface{}) error { return c.Cache.Set(key, value) } -//Get - get the value associated with the key +// Get - get the value associated with the key func (c *LRU) Get(key string) (interface{}, error) { value, err := c.Cache.Get(key) if err != nil { diff --git a/code/go/0chain.net/core/common/context.go b/code/go/0chain.net/core/common/context.go index 8d69fdcba..d7bf5a520 100644 --- a/code/go/0chain.net/core/common/context.go +++ b/code/go/0chain.net/core/common/context.go @@ -41,10 +41,10 @@ func GetRootContext() context.Context { /*Done - call this when the program needs to stop and notify all workers */ func Done() { - //Logger.Info("Initiating shutdown...") + // Logger.Info("Initiating shutdown...") rootCancel() - //TODO: How do we ensure every worker is completed any shutdown sequence before we finally shut down - //the server using server.Shutdown(ctx) + // TODO: How do we ensure every worker is completed any shutdown sequence before we finally shut down + // the server using server.Shutdown(ctx) } /*HandleShutdown - handles various shutdown signals */ diff --git a/code/go/0chain.net/core/common/errors.go b/code/go/0chain.net/core/common/errors.go index eede50d36..9f09e5b06 100644 --- a/code/go/0chain.net/core/common/errors.go +++ b/code/go/0chain.net/core/common/errors.go @@ -15,12 +15,12 @@ func (err *Error) Error() string { } /*NewError - create a new error */ -func NewError(code string, msg string) *Error { +func NewError(code, msg string) *Error { return &Error{Code: code, Msg: msg} } /*NewErrorf - create a new error with format */ -func NewErrorf(code string, format string, args ...interface{}) *Error { +func NewErrorf(code, format string, args ...interface{}) *Error { return &Error{Code: code, Msg: fmt.Sprintf(format, args...)} } diff --git a/code/go/0chain.net/core/common/handler.go b/code/go/0chain.net/core/common/handler.go index f84af02e6..14233ddac 100644 --- a/code/go/0chain.net/core/common/handler.go +++ b/code/go/0chain.net/core/common/handler.go @@ -22,7 +22,7 @@ const ( ClientSignatureHeader = "X-App-Client-Signature" ) -/*ReqRespHandlerf - a type for the default hanlder signature */ +/*ReqRespHandlerf - a type for the default handler signature */ type ReqRespHandlerf func(w http.ResponseWriter, r *http.Request) /*JSONResponderF - a handler that takes standard request (non-json) and responds with a json response @@ -50,18 +50,11 @@ func Respond(w http.ResponseWriter, data interface{}, err error) { buf := bytes.NewBuffer(nil) json.NewEncoder(buf).Encode(data) //nolint:errcheck // checked in previous step http.Error(w, buf.String(), 400) - } else { - if data != nil { - json.NewEncoder(w).Encode(data) //nolint:errcheck // checked in previous step - } + } else if data != nil { + json.NewEncoder(w).Encode(data) //nolint:errcheck // checked in previous step } } -func getContext(r *http.Request) (context.Context, error) { //nolint:unused,deadcode // might be used later? - ctx := r.Context() - return ctx, nil -} - var domainRE = regexp.MustCompile(`^(?:https?:\/\/)?(?:[^@\/\n]+@)?(?:www\.)?([^:\/\n]+)`) //nolint:unused,deadcode,varcheck // might be used later? func ToByteStream(handler JSONResponderF) ReqRespHandlerf { @@ -78,17 +71,14 @@ func ToByteStream(handler JSONResponderF) ReqRespHandlerf { } else { http.Error(w, err.Error(), 400) } - - } else { - if data != nil { - rawdata, ok := data.([]byte) - if ok { - w.Header().Set("Content-Type", "application/octet-stream") - w.Write(rawdata) //nolint:errcheck - } else { - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(data) //nolint:errcheck - } + } else if data != nil { + rawdata, ok := data.([]byte) + if ok { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(rawdata) //nolint:errcheck + } else { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(data) //nolint:errcheck } } } diff --git a/code/go/0chain.net/core/common/rate_limiter.go b/code/go/0chain.net/core/common/rate_limiter.go index d1201be9d..c5a16db5c 100644 --- a/code/go/0chain.net/core/common/rate_limiter.go +++ b/code/go/0chain.net/core/common/rate_limiter.go @@ -54,7 +54,6 @@ func (r *GRPCRateLimiter) Limit() bool { //UserRateLimit - rate limiting for end user handlers func UserRateLimit(handler ReqRespHandlerf) ReqRespHandlerf { - if !userRateLimit.RateLimit { return handler } diff --git a/code/go/0chain.net/core/common/time.go b/code/go/0chain.net/core/common/time.go index d35503638..0f64daf90 100644 --- a/code/go/0chain.net/core/common/time.go +++ b/code/go/0chain.net/core/common/time.go @@ -16,7 +16,7 @@ func Now() Timestamp { } /*Within ensures a given timestamp is within certain number of seconds */ -func Within(ts int64, seconds int64) bool { +func Within(ts, seconds int64) bool { now := time.Now().Unix() return now > ts-seconds && now < ts+seconds } diff --git a/code/go/0chain.net/core/common/utils.go b/code/go/0chain.net/core/common/utils.go index 606e2bc74..558bad211 100644 --- a/code/go/0chain.net/core/common/utils.go +++ b/code/go/0chain.net/core/common/utils.go @@ -2,23 +2,23 @@ package common import "fmt" -/*IsEmpty checks whether the input string is empty or not */ +// IsEmpty checks whether the input string is empty or not func IsEmpty(s string) bool { - return len(s) == 0 + return s == "" } -/*ToKey - takes an interface and returns a Key */ +// ToKey - takes an interface and returns a Key func ToKey(key interface{}) string { switch v := key.(type) { case string: - return string(v) + return v case []byte: return string(v) default: - return string(fmt.Sprintf("%v", v)) + return fmt.Sprintf("%v", v) } } -func IsEqual(key1 string, key2 string) bool { +func IsEqual(key1, key2 string) bool { return key1 == key2 } diff --git a/code/go/0chain.net/core/encryption/keys.go b/code/go/0chain.net/core/encryption/keys.go index 977db108d..bb652f1bf 100644 --- a/code/go/0chain.net/core/encryption/keys.go +++ b/code/go/0chain.net/core/encryption/keys.go @@ -15,7 +15,7 @@ import ( /*ReadKeys - reads a publicKey and a privateKey from a Reader. They are assumed to be in two separate lines one followed by the other*/ -func ReadKeys(reader io.Reader) (publicKey string, privateKey string, publicIp string, port string) { +func ReadKeys(reader io.Reader) (publicKey, privateKey, publicIp, port string) { scanner := bufio.NewScanner(reader) scanner.Scan() publicKey = scanner.Text() @@ -29,8 +29,8 @@ func ReadKeys(reader io.Reader) (publicKey string, privateKey string, publicIp s return publicKey, privateKey, publicIp, port } -//Verify - given a public key and a signature and the hash used to create the signature, verify the signature -func Verify(publicKey string, signature string, hash string) (bool, error) { +// Verify - given a public key and a signature and the hash used to create the signature, verify the signature +func Verify(publicKey, signature, hash string) (bool, error) { publicKey = MiraclToHerumiPK(publicKey) signature = MiraclToHerumiSig(signature) signScheme := zcncrypto.NewSignatureScheme(config.Configuration.SignatureScheme) diff --git a/code/go/0chain.net/core/lock/lock.go b/code/go/0chain.net/core/lock/lock.go index 25ca8ea9a..482963028 100644 --- a/code/go/0chain.net/core/lock/lock.go +++ b/code/go/0chain.net/core/lock/lock.go @@ -6,7 +6,7 @@ import ( ) var ( - // MutexCleanInterval start to clean unsed mutex at specified interval + // MutexCleanInterval start to clean unused mutex at specified interval MutexCleanInterval = 10 * time.Minute ) @@ -40,7 +40,7 @@ func (m *Mutex) Unlock() { } // GetMutex get mutex by table and key -func GetMutex(tablename string, key string) *Mutex { +func GetMutex(tablename, key string) *Mutex { lockKey := tablename + ":" + key lockMutex.Lock() @@ -76,8 +76,6 @@ func cleanUnusedMutexs() { func startWorker() { for { time.Sleep(MutexCleanInterval) - cleanUnusedMutexs() - } } diff --git a/code/go/0chain.net/core/lock/lock_test.go b/code/go/0chain.net/core/lock/lock_test.go index d78eaff07..cdcc276ad 100644 --- a/code/go/0chain.net/core/lock/lock_test.go +++ b/code/go/0chain.net/core/lock/lock_test.go @@ -8,11 +8,9 @@ import ( ) func TestLock(t *testing.T) { - max := 100 for i := 0; i < max; i++ { - lock1 := GetMutex("testlock", strconv.Itoa(i)) lock1.Lock() @@ -39,5 +37,4 @@ func TestLock(t *testing.T) { _, ok := lockPool["testlock:"+strconv.Itoa(i)] require.Equal(t, false, ok) } - } diff --git a/code/go/0chain.net/core/logging/logger.go b/code/go/0chain.net/core/logging/logger.go index 31bf5ccb3..1e73d9571 100644 --- a/code/go/0chain.net/core/logging/logger.go +++ b/code/go/0chain.net/core/logging/logger.go @@ -13,7 +13,7 @@ var ( Logger *zap.Logger ) -func InitLogging(mode string, logDir string, logFile string) { +func InitLogging(mode, logDir, logFile string) { var logName = logDir + "/" + logFile var logWriter = getWriteSyncer(logName) @@ -31,7 +31,6 @@ func InitLogging(mode string, logDir string, logFile string) { cfg.EncoderConfig.StacktraceKey = "stacktrace" logWriter = zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), logWriter) - } _ = cfg.Level.UnmarshalText([]byte(viper.GetString("logging.level"))) cfg.Encoding = "console" diff --git a/code/go/0chain.net/core/node/self_node.go b/code/go/0chain.net/core/node/self_node.go index 6c1a0ad51..be910587c 100644 --- a/code/go/0chain.net/core/node/self_node.go +++ b/code/go/0chain.net/core/node/self_node.go @@ -20,7 +20,7 @@ type SelfNode struct { } /*SetKeys - setter */ -func (sn *SelfNode) SetKeys(publicKey string, privateKey string) { +func (sn *SelfNode) SetKeys(publicKey, privateKey string) { publicKeyBytes, err := hex.DecodeString(publicKey) if err != nil { panic(err) diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index c6ea69311..57ba421de 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -8,7 +8,6 @@ import ( "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -88,7 +87,6 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] } for _, sharder := range selectedSharders { - u := fmt.Sprintf("%v/%v%v%v", sharder, SC_REST_API_URL, scAddress, relativePath) urls = append(urls, u+"?"+q.Encode()) @@ -110,7 +108,6 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] var msgList = make([]string, 1, numSharders) r := resty.New(transport, func(req *http.Request, resp *http.Response, cancelFunc context.CancelFunc, err error) error { - if err != nil { //network issue msgList = append(msgList, err.Error()) return err @@ -124,19 +121,17 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] msgList = append(msgList, errorMsg) return errors.Throw(ErrBadRequest, errorMsg) - } hash := fnv.New32() //use fnv for better performance teeReader := io.TeeReader(resp.Body, hash) - resBody, err := ioutil.ReadAll(teeReader) + resBody, err := io.ReadAll(teeReader) resp.Body.Close() if err != nil { errorMsg := "[sharder]body: " + url + " " + err.Error() msgList = append(msgList, errorMsg) return errors.Throw(ErrBadRequest, errorMsg) - } hashString := hex.EncodeToString(hash.Sum(nil)) @@ -172,16 +167,13 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] urls = []string{ fmt.Sprintf("%v/%v%v%v", network.Sharders[n], SC_REST_API_URL, scAddress, relativePath) + "?" + q.Encode(), } - } if hashMaxCounter < minNumConfirmation { - msgList[0] = fmt.Sprintf("min_confirmation is %v%%, but got %v/%v sharders", MinConfirmation, hashMaxCounter, numSharders) return nil, errors.Throw(ErrTooLessConfirmation, msgList...) } return resMaxCounterBody, nil - } diff --git a/code/go/0chain.net/core/util/http.go b/code/go/0chain.net/core/util/http.go index a1e983f33..b37ba0668 100644 --- a/code/go/0chain.net/core/util/http.go +++ b/code/go/0chain.net/core/util/http.go @@ -3,7 +3,7 @@ package util import ( "bytes" "context" - "io/ioutil" + "io" "net/http" "sync" "time" @@ -19,7 +19,7 @@ import ( const MAX_RETRIES = 5 const SLEEP_BETWEEN_RETRIES = 5 -func NewHTTPRequest(method string, url string, data []byte) (*http.Request, context.Context, context.CancelFunc, error) { +func NewHTTPRequest(method, url string, data []byte) (*http.Request, context.Context, context.CancelFunc, error) { requestHash := encryption.Hash(data) req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) req.Header.Set("Content-Type", "application/json; charset=utf-8") @@ -41,9 +41,7 @@ func SendMultiPostRequest(urls []string, data []byte) { wg.Wait() } -func SendPostRequest(url string, data []byte, wg *sync.WaitGroup) ( - body []byte, err error) { - +func SendPostRequest(url string, data []byte, wg *sync.WaitGroup) (body []byte, err error) { if wg != nil { defer wg.Done() } @@ -63,7 +61,7 @@ func SendPostRequest(url string, data []byte, wg *sync.WaitGroup) ( if resp.StatusCode >= 200 && resp.StatusCode <= 299 { break } - body, _ = ioutil.ReadAll(resp.Body) + body, _ = io.ReadAll(resp.Body) if resp.Body != nil { resp.Body.Close() } @@ -83,6 +81,6 @@ func SendPostRequest(url string, data []byte, wg *sync.WaitGroup) ( defer resp.Body.Close() } - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) return body, err } diff --git a/code/go/0chain.net/validator/main.go b/code/go/0chain.net/validator/main.go index 413d647d3..35541ea8d 100644 --- a/code/go/0chain.net/validator/main.go +++ b/code/go/0chain.net/validator/main.go @@ -162,7 +162,6 @@ func main() { } func RegisterValidator() { - registrationRetries := 0 //ctx := badgerdbstore.GetStorageProvider().WithConnection(common.GetRootContext()) for registrationRetries < 10 { diff --git a/code/go/0chain.net/validatorcore/config/config.go b/code/go/0chain.net/validatorcore/config/config.go index 4b59c4bd7..7d100099b 100644 --- a/code/go/0chain.net/validatorcore/config/config.go +++ b/code/go/0chain.net/validatorcore/config/config.go @@ -7,7 +7,7 @@ import ( "github.com/spf13/viper" ) -//SetupDefaultConfig - setup the default config options that can be overridden via the config file +// SetupDefaultConfig - setup the default config options that can be overridden via the config file func SetupDefaultConfig() { viper.SetDefault("logging.level", "info") viper.SetDefault("delegate_wallet", "") diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler.go b/code/go/0chain.net/validatorcore/storage/challenge_handler.go index 28add0fd9..2222e2350 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler.go @@ -109,5 +109,4 @@ func ChallengeHandler(ctx context.Context, r *http.Request) (interface{}, error) lru.Add(challengeHash, &validationTicket) //nolint:errcheck // never returns an error anyway return &validationTicket, nil - } diff --git a/code/go/0chain.net/validatorcore/storage/models.go b/code/go/0chain.net/validatorcore/storage/models.go index a5f93d0fb..59ca96e83 100644 --- a/code/go/0chain.net/validatorcore/storage/models.go +++ b/code/go/0chain.net/validatorcore/storage/models.go @@ -68,7 +68,6 @@ func (r *DirMetaData) GetHash() string { } func (r *DirMetaData) CalculateHash() string { - childHashes := make([]string, len(r.Children)) for index, childRef := range r.Children { childHashes[index] = childRef.GetHash() @@ -98,17 +97,19 @@ type FileMetaData struct { func (fr *FileMetaData) GetHashData() string { hashArray := make([]string, 0) - hashArray = append(hashArray, fr.AllocationID) - hashArray = append(hashArray, fr.Type) - hashArray = append(hashArray, fr.Name) - hashArray = append(hashArray, fr.Path) - hashArray = append(hashArray, strconv.FormatInt(fr.Size, 10)) - hashArray = append(hashArray, fr.ContentHash) - hashArray = append(hashArray, fr.MerkleRoot) - hashArray = append(hashArray, strconv.FormatInt(fr.ActualFileSize, 10)) - hashArray = append(hashArray, fr.ActualFileHash) - hashArray = append(hashArray, fr.Attributes.String()) - hashArray = append(hashArray, strconv.FormatInt(fr.ChunkSize, 10)) + hashArray = append(hashArray, + fr.AllocationID, + fr.Type, + fr.Name, + fr.Path, + strconv.FormatInt(fr.Size, 10), + fr.ContentHash, + fr.MerkleRoot, + strconv.FormatInt(fr.ActualFileSize, 10), + fr.ActualFileHash, + fr.Attributes.String(), + strconv.FormatInt(fr.ChunkSize, 10), + ) return strings.Join(hashArray, ":") } @@ -207,7 +208,7 @@ func (op *ObjectPath) VerifyBlockNum(challengeRand int64) error { r := rand.New(rand.NewSource(challengeRand)) //rand.Seed(challengeRand) blockNum := r.Int63n(op.RootObject.NumBlocks) - blockNum = blockNum + 1 + blockNum++ if op.RootObject.NumBlocks < blockNum { return common.NewError("invalid_block_num", fmt.Sprintf("Invalid block number %d/%d", op.RootObject.NumBlocks, blockNum)) @@ -224,7 +225,7 @@ func (op *ObjectPath) VerifyBlockNum(challengeRand int64) error { } for _, child := range curRef.(*DirMetaData).Children { if child.GetNumBlocks() < remainingBlocks { - remainingBlocks = remainingBlocks - child.GetNumBlocks() + remainingBlocks -= child.GetNumBlocks() continue } if child.GetType() == FILE { diff --git a/code/go/0chain.net/validatorcore/storage/protocol.go b/code/go/0chain.net/validatorcore/storage/protocol.go index 192027a3d..9d91977e4 100644 --- a/code/go/0chain.net/validatorcore/storage/protocol.go +++ b/code/go/0chain.net/validatorcore/storage/protocol.go @@ -83,7 +83,7 @@ func (sp *ValidatorProtocolImpl) VerifyAllocationTransaction(ctx context.Context func (sp *ValidatorProtocolImpl) VerifyChallengeTransaction(ctx context.Context, challengeRequest *ChallengeRequest) (*Challenge, error) { blobberID := ctx.Value(constants.ContextKeyClient).(string) - if len(blobberID) == 0 { + if blobberID == "" { return nil, common.NewError("invalid_client", "Call from an invalid client") } params := make(map[string]string) @@ -122,13 +122,12 @@ type WalletCallback struct { err string } -func (wb *WalletCallback) OnWalletCreateComplete(status int, wallet string, err string) { +func (wb *WalletCallback) OnWalletCreateComplete(status int, wallet, err string) { wb.err = err wb.wg.Done() } func (sp *ValidatorProtocolImpl) RegisterValidator(ctx context.Context) (string, error) { - wcb := &WalletCallback{} wcb.wg = &sync.WaitGroup{} wcb.wg.Add(1) diff --git a/code/go/0chain.net/validatorcore/storage/writemarker/entity.go b/code/go/0chain.net/validatorcore/storage/writemarker/entity.go index b5850d1eb..bbae60270 100644 --- a/code/go/0chain.net/validatorcore/storage/writemarker/entity.go +++ b/code/go/0chain.net/validatorcore/storage/writemarker/entity.go @@ -44,7 +44,7 @@ func (wm *WriteMarker) VerifySignature(clientPublicKey string) bool { return true } -func (wm *WriteMarker) Verify(allocationID string, allocationRoot string, clientPublicKey string) error { +func (wm *WriteMarker) Verify(allocationID, allocationRoot, clientPublicKey string) error { if wm.AllocationID != allocationID { return common.NewError("challenge_validation_failed", "Invalid write marker. Allocation ID mismatch") } From 557ee8753341a2397e83cfa07ce5d344e004bce7 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Fri, 4 Feb 2022 10:01:58 +0545 Subject: [PATCH 097/161] Optimize/verify auth ticket function (#525) * Optimize auth ticket verification function * Fix mock --- .../0chain.net/blobbercore/handler/handler.go | 4 +- .../blobbercore/handler/handler_test.go | 2 +- .../handler/object_operation_handler.go | 7 +-- .../blobbercore/handler/storage_handler.go | 52 +++++++------------ 4 files changed, 25 insertions(+), 40 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index e1c90b8f7..7796a3c39 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -479,8 +479,8 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } - authTicketVerified, err := storageHandler.verifyAuthTicket(ctx, authTicketString, allocationObj, fileref, authTicket.ClientID) - if !authTicketVerified { + authToken, err := storageHandler.verifyAuthTicket(ctx, authTicketString, allocationObj, fileref, authTicket.ClientID) + if authToken == nil { return nil, common.NewError("auth_ticket_verification_failed", "Could not verify the auth ticket. "+err.Error()) } diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 3149122e9..498216b47 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -2145,7 +2145,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { WithArgs(alloc.ID, filePathHash). WillReturnRows( sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder1/subfolder1", filestore.CHUNK_SIZE), + AddRow("/folder1/subfolder1/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder1/subfolder1", filestore.CHUNK_SIZE), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 32d48d1f8..5aca5f898 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -283,11 +283,8 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r var authTokenString = r.FormValue("auth_token") // check auth token - if isAuthorized, err := fsh.verifyAuthTicket(ctx, - authTokenString, alloc, fileref, clientID, - ); !isAuthorized { - return nil, common.NewErrorf("download_file", - "cannot verify auth ticket: %v", err) + if authToken, err := fsh.verifyAuthTicket(ctx, authTokenString, alloc, fileref, clientID); authToken == nil { + return nil, common.NewErrorf("download_file", "cannot verify auth ticket: %v", err) } authToken = &readmarker.AuthTicket{} diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 3c2b8c912..36415f0ae 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -8,7 +8,6 @@ import ( "net/http" "regexp" "strconv" - "strings" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "gorm.io/gorm" @@ -66,41 +65,33 @@ func (fsh *StorageHandler) convertGormError(err error) error { return err } -func (fsh *StorageHandler) verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string) (bool, error) { +// verifyAuthTicket verifies authTicket and returns authToken and error if any. For any error authToken is nil +func (fsh *StorageHandler) verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string) (*readmarker.AuthTicket, error) { if authTokenString == "" { - return false, common.NewError("invalid_parameters", "Auth ticket required if data read by anyone other than owner.") + return nil, common.NewError("invalid_parameters", "Auth ticket is required") } + authToken := &readmarker.AuthTicket{} - err := json.Unmarshal([]byte(authTokenString), &authToken) - if err != nil { - return false, common.NewError("invalid_parameters", "Error parsing the auth ticket for download."+err.Error()) + if err := json.Unmarshal([]byte(authTokenString), &authToken); err != nil { + return nil, common.NewError("invalid_parameters", "Error parsing the auth ticket for download."+err.Error()) } - err = authToken.Verify(allocationObj, clientID) - if err != nil { - return false, err + + if err := authToken.Verify(allocationObj, clientID); err != nil { + return nil, err } + if refRequested.LookupHash != authToken.FilePathHash { authTokenRef, err := reference.GetReferenceFromLookupHash(ctx, authToken.AllocationID, authToken.FilePathHash) if err != nil { - return false, err - } - - parentPath := refRequested.ParentPath - if !strings.HasPrefix(parentPath, "/") { - parentPath = "/" + parentPath - } - - authRefPath := authTokenRef.Path - if strings.HasPrefix(authRefPath, ".") || authRefPath == "/" { - authRefPath = "" + return nil, err } - if refRequested.ParentPath != authTokenRef.Path && !strings.HasPrefix(parentPath, authRefPath+"/") { - return false, common.NewError("invalid_parameters", "Auth ticket is not valid for the resource being requested") + if matched, _ := regexp.MatchString(fmt.Sprintf("^%v", authTokenRef.Path), refRequested.Path); !matched { + return nil, common.NewError("invalid_parameters", "Auth ticket is not valid for the resource being requested") } } - return true, nil + return authToken, nil } func (fsh *StorageHandler) GetAllocationDetails(ctx context.Context, r *http.Request) (interface{}, error) { @@ -206,11 +197,8 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in var authTokenString = r.FormValue("auth_token") // check auth token - if isAuthorized, err := fsh.verifyAuthTicket(ctx, - authTokenString, alloc, fileref, clientID, - ); !isAuthorized { - return nil, common.NewErrorf("file_meta", - "cannot verify auth ticket: %v", err) + if authToken, err := fsh.verifyAuthTicket(ctx, authTokenString, alloc, fileref, clientID); authToken == nil { + return nil, common.NewErrorf("file_meta", "cannot verify auth ticket: %v", err) } delete(result, "path") @@ -255,11 +243,11 @@ func (fsh *StorageHandler) AddCommitMetaTxn(ctx context.Context, r *http.Request authTokenString := r.FormValue("auth_token") if clientID != allocationObj.OwnerID || len(authTokenString) > 0 { - authTicketVerified, err := fsh.verifyAuthTicket(ctx, r.FormValue("auth_token"), allocationObj, fileref, clientID) + authToken, err := fsh.verifyAuthTicket(ctx, r.FormValue("auth_token"), allocationObj, fileref, clientID) if err != nil { return nil, err } - if !authTicketVerified { + if authToken == nil { return nil, common.NewError("auth_ticket_verification_failed", "Could not verify the auth ticket.") } } @@ -455,11 +443,11 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* authTokenString := r.FormValue("auth_token") if clientID != allocationObj.OwnerID || len(authTokenString) > 0 { - authTicketVerified, err := fsh.verifyAuthTicket(ctx, r.FormValue("auth_token"), allocationObj, fileref, clientID) + authToken, err := fsh.verifyAuthTicket(ctx, r.FormValue("auth_token"), allocationObj, fileref, clientID) if err != nil { return nil, err } - if !authTicketVerified { + if authToken == nil { return nil, common.NewError("auth_ticket_verification_failed", "Could not verify the auth ticket.") } } From 60edfeeccac0c78213bb61491467f41ae4b161c8 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Sat, 5 Feb 2022 14:01:32 +0545 Subject: [PATCH 098/161] Fix/public share issue (#528) * Fix expiration issue, add default 90 days to public share * Remove redundant and erroneous checking * Modify . import issue * Remove redundant allocationID retrieval * Move condition checking --- .../0chain.net/blobbercore/handler/handler.go | 31 +++++++------------ .../blobbercore/readmarker/entity.go | 24 ++++++++++---- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 7796a3c39..07c3d2907 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -405,11 +405,8 @@ func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { } sign := r.Header.Get(common.ClientSignatureHeader) - allocation, ok := mux.Vars(r)["allocation"] - if !ok { - return false, common.NewError("invalid_params", "Missing allocation tx") - } - valid, err := verifySignatureFromRequest(allocation, sign, allocationObj.OwnerPublicKey) + + valid, err := verifySignatureFromRequest(allocationID, sign, allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } @@ -421,10 +418,12 @@ func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } + clientID := ctx.Value(constants.ContextKeyClient).(string) if clientID != allocationObj.OwnerID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } + err = reference.DeleteShareInfo(ctx, &reference.ShareInfo{ ClientID: refereeClientID, FilePathHash: filePathHash, @@ -436,9 +435,11 @@ func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { } return resp, nil } + if err != nil { return nil, err } + resp := map[string]interface{}{ "status": http.StatusNoContent, "message": "Path successfully removed from allocation", @@ -456,11 +457,8 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { } sign := r.Header.Get(common.ClientSignatureHeader) - allocation, ok := mux.Vars(r)["allocation"] - if !ok { - return false, common.NewError("invalid_params", "Missing allocation tx") - } - valid, err := verifySignatureFromRequest(allocation, sign, allocationObj.OwnerPublicKey) + + valid, err := verifySignatureFromRequest(allocationID, sign, allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } @@ -488,11 +486,6 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { return nil, err } - // dummy, to avoid input and sql error - if len(authTicket.ClientID) != 64 || len(authTicket.OwnerID) != 64 { - return nil, common.NewError("share_info_insert", "Wrong ownerID or clientID") - } - shareInfo := reference.ShareInfo{ OwnerID: authTicket.OwnerID, ClientID: authTicket.ClientID, @@ -509,15 +502,13 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { } else { err = reference.AddShareInfo(ctx, shareInfo) } + if err != nil { + Logger.Info(err.Error()) return nil, common.NewError("share_info_insert", "Unable to save share info") } - resp := map[string]interface{}{ - "message": "Share info added successfully", - } - - return resp, nil + return map[string]interface{}{"message": "Share info added successfully"}, nil } func MarketPlaceShareInfoHandler(ctx context.Context, r *http.Request) (interface{}, error) { diff --git a/code/go/0chain.net/blobbercore/readmarker/entity.go b/code/go/0chain.net/blobbercore/readmarker/entity.go index f0e6a20de..1728344d5 100644 --- a/code/go/0chain.net/blobbercore/readmarker/entity.go +++ b/code/go/0chain.net/blobbercore/readmarker/entity.go @@ -4,16 +4,20 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + zLogger "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/datatypes" +) - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "go.uber.org/zap" +const ( + NinetyDays = common.Timestamp(90 * 24 * time.Hour) ) type AuthTicket struct { @@ -40,11 +44,18 @@ func (authToken *AuthTicket) Verify(allocationObj *allocation.Allocation, client if authToken.AllocationID != allocationObj.ID { return common.NewError("invalid_parameters", "Invalid auth ticket. Allocation id mismatch") } - if authToken.ClientID != clientID && len(authToken.ClientID) > 0 { + if authToken.ClientID != "" && authToken.ClientID != clientID { return common.NewError("invalid_parameters", "Invalid auth ticket. Client ID mismatch") } - if authToken.Expiration > 0 && (authToken.Expiration < authToken.Timestamp || authToken.Expiration < common.Now()) { - return common.NewError("invalid_parameters", "Invalid auth ticket. Expired ticket") + + if authToken.Expiration > 0 { + if authToken.Expiration < authToken.Timestamp || authToken.Expiration <= common.Now() { + return common.NewError("invalid_parameters", "Invalid auth ticket. Expired ticket") + } + } else { // check for default 90 days expiration time + if authToken.Timestamp+NinetyDays <= common.Now() { + return common.NewError("invalid_parameters", "Authticket expired") + } } if authToken.OwnerID != allocationObj.OwnerID { @@ -60,6 +71,7 @@ func (authToken *AuthTicket) Verify(allocationObj *allocation.Allocation, client if err != nil || !sigOK { return common.NewError("invalid_parameters", "Invalid auth ticket. Signature verification failed") } + return nil } @@ -169,7 +181,7 @@ func (rm *ReadMarkerEntity) Sync(ctx context.Context) (err error) { func (rm *ReadMarkerEntity) UpdateStatus(ctx context.Context, rps []*allocation.ReadPool, txOutput, redeemTxn string) (err error) { var redeems []allocation.ReadPoolRedeem if err = json.Unmarshal([]byte(txOutput), &redeems); err != nil { - Logger.Error("update read redeeming status: can't decode transaction"+ + zLogger.Logger.Error("update read redeeming status: can't decode transaction"+ " output", zap.Error(err)) return common.NewErrorf("rme_update_status", "can't decode transaction output: %v", err) From 629a907e167e055b86426137b80891dbf92b26f7 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Tue, 8 Feb 2022 12:49:31 +0545 Subject: [PATCH 099/161] Allow collaborators to provide share to other clients (#531) * Allow collaborators to provide share to other clients * Allow only owners to share file check if client is same as allocation owner * Remove double checking clientid --- code/go/0chain.net/blobbercore/handler/handler.go | 10 +++++++++- code/go/0chain.net/blobbercore/handler/handler_test.go | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 07c3d2907..fa76a2fa1 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -450,7 +450,11 @@ func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { ctx = setupHandlerContext(ctx, r) - allocationID := ctx.Value(constants.ContextKeyAllocation).(string) + var ( + allocationID = ctx.Value(constants.ContextKeyAllocation).(string) + clientID = ctx.Value(constants.ContextKeyClient).(string) + ) + allocationObj, err := storageHandler.verifyAllocation(ctx, allocationID, true) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) @@ -463,6 +467,10 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { return nil, common.NewError("invalid_signature", "Invalid signature") } + if clientID != allocationObj.OwnerID { + return nil, common.NewError("invalid_client", "Client has no access to share file") + } + encryptionPublicKey := r.FormValue("encryption_public_key") authTicketString := r.FormValue("auth_ticket") authTicket := &readmarker.AuthTicket{} diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 498216b47..4eb32e4cf 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -1148,6 +1148,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { r.Header.Set("Content-Type", formWriter.FormDataContentType()) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) return r }(), @@ -1236,6 +1237,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { r.Header.Set("Content-Type", formWriter.FormDataContentType()) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) return r }(), From 743d0d48fe12e9a74e69ffee2eddb5d40e3065f1 Mon Sep 17 00:00:00 2001 From: prince Date: Thu, 10 Feb 2022 09:47:41 +0530 Subject: [PATCH 100/161] write-marker for copied files. (#492) * write-marker for copied files. Signed-off-by: Pradip Parmar * error check for newdir stats. Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar --- .../blobbercore/allocation/copyfilechange.go | 23 +++++++++++++++++++ .../blobbercore/allocation/newfilechange.go | 5 +++- .../0chain.net/blobbercore/stats/filestats.go | 4 ++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 41e4904e6..0851c5592 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/core/common" "gorm.io/datatypes" @@ -95,10 +96,32 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC dirRef.AddChild(destRef) _, err = rootRef.CalculateHash(ctx, true) + if err != nil { + return nil, err + } + + err = rf.updateWriteMarker(ctx, destRef, affectedRef) return rootRef, err } +func (rf *CopyFileChange) updateWriteMarker(ctx context.Context, destRef, affectedRef *reference.Ref) error { + ref := destRef + if affectedRef != nil { + for _, r := range destRef.Children { + if affectedRef.Name == r.Name { + ref = r + } + } + } + + if ref.Type == reference.FILE { + return stats.NewDirCreated(ctx, ref.ID) + } + + return rf.updateWriteMarker(ctx, ref, nil) +} + func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, destRef *reference.Ref, allocationRoot string) { if affectedRef.Type == reference.DIRECTORY { newRef := reference.NewDirectoryRef() diff --git a/code/go/0chain.net/blobbercore/allocation/newfilechange.go b/code/go/0chain.net/blobbercore/allocation/newfilechange.go index c4e05e678..aa708caba 100644 --- a/code/go/0chain.net/blobbercore/allocation/newfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/newfilechange.go @@ -111,7 +111,10 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a return nil, err } - stats.NewDirCreated(ctx, dirRef.ID) + if err := stats.NewDirCreated(ctx, dirRef.ID); err != nil { + return nil, err + } + return rootRef, nil } diff --git a/code/go/0chain.net/blobbercore/stats/filestats.go b/code/go/0chain.net/blobbercore/stats/filestats.go index eb17c8e15..713664e95 100644 --- a/code/go/0chain.net/blobbercore/stats/filestats.go +++ b/code/go/0chain.net/blobbercore/stats/filestats.go @@ -23,12 +23,12 @@ func (FileStats) TableName() string { return "file_stats" } -func NewDirCreated(ctx context.Context, refID int64) { +func NewDirCreated(ctx context.Context, refID int64) error { db := datastore.GetStore().GetTransaction(ctx) stats := &FileStats{RefID: refID} stats.NumBlockDownloads = 0 stats.NumUpdates = 1 - db.Save(stats) + return db.Save(stats).Error } func NewFileCreated(ctx context.Context, refID int64) { From 73e5db1bdf371c0f57a660673a42528482bb238f Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Thu, 10 Feb 2022 13:44:37 +0545 Subject: [PATCH 101/161] Optimize/download (#526) * Optimize auth ticket verification function * Fix mock * Reduce code * Optimize redundant code * Upgrade gosdk * Make authticket mandatory to other clients Modify tests accordingly * Allow collaborator to download file without authticket * Fix collaborator file download test * Fix Checking clientid from authtoken --- .../blobbercore/handler/handler_test.go | 345 +++++++++++------- .../handler/object_operation_handler.go | 190 ++++------ .../handler/object_operation_handler_test.go | 119 +++++- .../blobbercore/handler/storage_handler.go | 16 +- go.mod | 2 +- go.sum | 4 +- 6 files changed, 389 insertions(+), 287 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 4eb32e4cf..a960b0a32 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -45,12 +45,7 @@ type MockFileBlockGetter struct { var mockFileBlock []byte -func (MockFileBlockGetter) GetFileBlock( - fsStore *filestore.FileFSStore, - allocationID string, - fileData *filestore.FileInputData, - blockNum int64, - numBlocks int64) ([]byte, error) { +func (MockFileBlockGetter) GetFileBlock(fsStore *filestore.FileFSStore, allocationID string, fileData *filestore.FileInputData, blockNum, numBlocks int64) ([]byte, error) { return mockFileBlock, nil } @@ -62,15 +57,44 @@ func resetMockFileBlock() { mockFileBlock = []byte("mock") } -var encscheme zencryption.EncryptionScheme +// var encscheme zencryption.EncryptionScheme + +// func setupEncryptionScheme() { +// encscheme = zencryption.NewEncryptionScheme() +// mnemonic := client.GetClient().Mnemonic +// if _, err := encscheme.Initialize(mnemonic); err != nil { +// panic("initialize encscheme") +// } +// encscheme.InitForEncryption("filetype:audio") +// } + +func signHash(client *client.Client, hash string) (string, error) { + retSignature := "" + for _, kv := range client.Keys { + ss := zcncrypto.NewSignatureScheme("bls0chain") + err := ss.SetPrivateKey(kv.PrivateKey) + if err != nil { + return "", err + } + if len(retSignature) == 0 { + retSignature, err = ss.Sign(hash) + } else { + retSignature, err = ss.Add(retSignature, hash) + } + if err != nil { + return "", err + } + } + return retSignature, nil +} -func setupEncryptionScheme() { - encscheme = zencryption.NewEncryptionScheme() - mnemonic := client.GetClient().Mnemonic +func getEncryptionScheme(mnemonic string) (zencryption.EncryptionScheme, error) { + encscheme := zencryption.NewEncryptionScheme() if _, err := encscheme.Initialize(mnemonic); err != nil { - panic("initialize encscheme") + return nil, err } encscheme.InitForEncryption("filetype:audio") + return encscheme, nil } func init() { @@ -128,8 +152,8 @@ func setup(t *testing.T) { } } -func setupHandlers() (router *mux.Router, opMap map[string]string) { - router = mux.NewRouter() +func setupHandlers() (*mux.Router, map[string]string) { + router := mux.NewRouter() opPath := "/v1/file/objectpath/{allocation}" opName := "Object_Path" @@ -264,10 +288,10 @@ func isEndpointAllowGetReq(name string) bool { } } -func GetAuthTicketForEncryptedFile(allocationID, remotePath, fileHash, clientID, encPublicKey string) (string, error) { +func GetAuthTicketForEncryptedFile(ownerClient *client.Client, allocationID, remotePath, fileHash, clientID, encPublicKey string) (string, error) { at := &marker.AuthTicket{} at.AllocationID = allocationID - at.OwnerID = client.GetClientID() + at.OwnerID = ownerClient.ClientID at.ClientID = clientID at.FileName = remotePath at.FilePathHash = fileHash @@ -280,7 +304,11 @@ func GetAuthTicketForEncryptedFile(allocationID, remotePath, fileHash, clientID, at.Expiration = timestamp + 7776000 at.Timestamp = timestamp at.ReEncryptionKey = "regenkey" - err := at.Sign() + at.Encrypted = true + + hash := encryption.Hash(at.GetHashData()) + var err error + at.Signature, err = signHash(ownerClient, hash) if err != nil { return "", err } @@ -294,21 +322,39 @@ func GetAuthTicketForEncryptedFile(allocationID, remotePath, fileHash, clientID, func TestHandlers_Requiring_Signature(t *testing.T) { setup(t) - clientJson := "{\"client_id\":\"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9\",\"client_key\":\"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083\",\"keys\":[{\"public_key\":\"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083\",\"private_key\":\"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907\"}],\"mnemonics\":\"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe\",\"version\":\"1.0\",\"date_created\":\"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805\"}" - require.NoError(t, client.PopulateClient(clientJson, "bls0chain")) - setupEncryptionScheme() + clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}` + guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}` + + require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) + clients := client.GetClients() + + ownerClient, guestClient := clients[0], clients[1] + + ownerScheme, err := getEncryptionScheme(ownerClient.Mnemonic) + if err != nil { + t.Fatal(err) + } + + guestScheme, err := getEncryptionScheme(guestClient.Mnemonic) + if err != nil { + t.Fatal(err) + } + // require.NoError(t, client.PopulateClient(clientJson, "bls0chain")) + // setupEncryptionScheme() + router, handlers := setupHandlers() sch := zcncrypto.NewSignatureScheme("bls0chain") //sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" - _, err := sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") + _, err = sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") if err != nil { t.Fatal(err) } + ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) - alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerPublicKey = ownerClient.Keys[0].PublicKey + alloc.OwnerID = ownerClient.ClientID const ( path = "/path" @@ -337,6 +383,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { if !isEndpointRequireSignature(name) { continue } + baseSetupDbMock := func(mock sqlmock.Sqlmock) { mock.ExpectBegin() @@ -371,7 +418,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { if !isEndpointAllowGetReq(name) { method = http.MethodPost } - r, err := http.NewRequest(method, url.String(), http.NoBody) + r, err := http.NewRequest(method, url.String(), nil) if err != nil { t.Fatal(err) } @@ -400,7 +447,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { if !isEndpointAllowGetReq(name) { method = http.MethodPost } - r, err := http.NewRequest(method, url.String(), http.NoBody) + r, err := http.NewRequest(method, url.String(), nil) if err != nil { t.Fatal(err) } @@ -441,7 +488,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) } @@ -501,7 +548,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) } @@ -561,7 +608,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodPost, url.String(), nil) if err != nil { t.Fatal(err) } @@ -632,7 +679,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) } @@ -693,7 +740,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("collab_id", "collab id") url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodGet, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) } @@ -762,7 +809,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("connection_id", connectionID) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodPost, url.String(), nil) if err != nil { t.Fatal(err) } @@ -842,7 +889,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("dest", "dest") url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodPost, url.String(), nil) if err != nil { t.Fatal(err) } @@ -933,7 +980,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("attributes", string(attrBytes)) url.RawQuery = q.Encode() - r, err := http.NewRequest(http.MethodPost, url.String(), http.NoBody) + r, err := http.NewRequest(http.MethodPost, url.String(), nil) if err != nil { t.Fatal(err) } @@ -1125,7 +1172,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, formWriter.WriteField("encryption_public_key", shareClientEncryptionPublicKey)) remotePath := "/file.txt" filePathHash := "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c" - authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, filePathHash, shareClientID, sch.GetPublicKey()) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, filePathHash, shareClientID, ownerClient.Keys[0].PublicKey) if err != nil { t.Fatal(err) } @@ -1190,7 +1237,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { aa := sqlmock.AnyArg() mock.ExpectExec(`INSERT INTO "marketplace_share_info"`). - WithArgs(client.GetClientID(), "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c", "regenkey", aa, false, aa). + WithArgs("2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9", "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c", "regenkey", aa, false, aa). WillReturnResult(sqlmock.NewResult(0, 0)) }, wantCode: http.StatusOK, @@ -1214,7 +1261,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, formWriter.WriteField("encryption_public_key", shareClientEncryptionPublicKey)) remotePath := "/file.txt" filePathHash := "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c" - authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, filePathHash, shareClientID, sch.GetPublicKey()) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, filePathHash, shareClientID, sch.GetPublicKey()) if err != nil { t.Fatal(err) } @@ -1363,6 +1410,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). WithArgs(true, "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", filePathHash). WillReturnResult(sqlmock.NewResult(0, 1)) + }, wantCode: http.StatusOK, wantBody: "{\"message\":\"Path successfully removed from allocation\",\"status\":204}\n", @@ -1442,6 +1490,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). WithArgs(true, "da4b54d934890aa415bb043ce1126f2e30a96faf63a4c65c25bbddcb32824d77", filePathHash). WillReturnResult(sqlmock.NewResult(0, 0)) + }, wantCode: http.StatusOK, wantBody: "{\"message\":\"Path not found\",\"status\":404}\n", @@ -1464,12 +1513,12 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, formWriter.WriteField("path_hash", fileref.GetReferenceLookup(alloc.Tx, remotePath))) require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() + rm.ClientID = ownerClient.ClientID + rm.ClientPublicKey = ownerClient.ClientKey rm.BlobberID = "" rm.AllocationID = alloc.ID - rm.OwnerID = client.GetClientID() - err = rm.Sign() + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(ownerClient, rm.GetHash()) if err != nil { t.Fatal(err) } @@ -1526,6 +1575,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, filePathHash).WillReturnError(gorm.ErrRecordNotFound) + }, wantCode: http.StatusBadRequest, wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: invalid file path: record not found\"}\n\n", @@ -1548,13 +1598,13 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, formWriter.WriteField("path_hash", fileref.GetReferenceLookup(alloc.Tx, remotePath))) require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() + rm.ClientID = ownerClient.ClientID + rm.ClientPublicKey = ownerClient.ClientKey rm.BlobberID = "" rm.AllocationID = alloc.ID rm.ReadCounter = 1 - rm.OwnerID = client.GetClientID() - err = rm.Sign() + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(ownerClient, rm.GetHash()) if err != nil { t.Fatal(err) } @@ -1617,20 +1667,20 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(ownerClient.ClientID). WillReturnError(gorm.ErrRecordNotFound) mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(ownerClient.ClientID). WillReturnRows( sqlmock.NewRows([]string{"client_id"}). - AddRow(client.GetClientID()), + AddRow(ownerClient.ClientID), ) aa := sqlmock.AnyArg() mock.ExpectExec(`(UPDATE "read_markers" SET)(.+)`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WithArgs(ownerClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -1656,20 +1706,20 @@ func TestHandlers_Requiring_Signature(t *testing.T) { pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) require.NoError(t, formWriter.WriteField("path_hash", pathHash)) require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, ownerClient.Keys[0].PublicKey) if err != nil { t.Fatal(err) } require.NoError(t, formWriter.WriteField("auth_token", authTicket)) rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey rm.BlobberID = "" rm.AllocationID = alloc.ID rm.ReadCounter = 1 - rm.OwnerID = client.GetClientID() - err = rm.Sign() + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(guestClient, rm.GetHash()) if err != nil { t.Fatal(err) } @@ -1693,8 +1743,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { r.Header.Set("Content-Type", formWriter.FormDataContentType()) r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) return r }(), @@ -1776,19 +1826,19 @@ func TestHandlers_Requiring_Signature(t *testing.T) { pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) require.NoError(t, formWriter.WriteField("path_hash", pathHash)) require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") if err != nil { t.Fatal(err) } require.NoError(t, formWriter.WriteField("auth_token", authTicket)) rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey rm.BlobberID = "" rm.AllocationID = alloc.ID rm.ReadCounter = 1 - rm.OwnerID = client.GetClientID() - err = rm.Sign() + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(guestClient, rm.GetHash()) if err != nil { t.Fatal(err) } @@ -1812,8 +1862,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { r.Header.Set("Content-Type", formWriter.FormDataContentType()) r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) return r }(), @@ -1821,13 +1871,15 @@ func TestHandlers_Requiring_Signature(t *testing.T) { alloc: alloc, begin: func() { dataToEncrypt := "data_to_encrypt" - encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) if err != nil { t.Fatal(err) } - header := make([]byte, 2*1024) + + header := make([]byte, HeaderChecksumSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) + fmt.Println("Encrypted data: ", string(data)) setMockFileBlock(data) }, end: func() { @@ -1861,34 +1913,43 @@ func TestHandlers_Requiring_Signature(t *testing.T) { WithArgs(alloc.ID, filePathHash). WillReturnRows( sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), 65536), + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), 65536), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(guestClient.ClientID). WillReturnError(gorm.ErrRecordNotFound) + guestPublicEncryptedKey, err := guestScheme.GetPublicKey() + if err != nil { + t.Fatal(err) + } + reEncryptionKey, err := ownerScheme.GetReGenKey(guestPublicEncryptedKey, "filetype:audio") + + if err != nil { + t.Fatal(err) + } + fmt.Printf("\n\nencryptedKey: %v\tgpbk: %v\treKey: %v\n\n", ownerScheme.GetEncryptedKey(), guestPublicEncryptedKey, reEncryptionKey) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(guestClient.ClientID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, guestPublicEncryptedKey), + ) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(guestClient.ClientID). WillReturnRows( sqlmock.NewRows([]string{"client_id"}). - AddRow(client.GetClientID()), + AddRow(guestClient.ClientID), ) aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). + WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) - reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs(client.GetClientID(), filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). - AddRow(reEncryptionKey, encscheme.GetEncryptedKey()), - ) - mock.ExpectCommit() }, wantCode: http.StatusOK, @@ -1915,28 +1976,30 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") if err != nil { t.Fatal(err) } require.NoError(t, formWriter.WriteField("auth_token", authTicket)) rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey rm.BlobberID = "" rm.AllocationID = alloc.ID rm.ReadCounter = 1 - rm.OwnerID = client.GetClientID() - err = rm.Sign() + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(guestClient, rm.GetHash()) if err != nil { t.Fatal(err) } + rmData, err := json.Marshal(rm) require.NoError(t, err) require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) if err := formWriter.Close(); err != nil { t.Fatal(err) } + r, err := http.NewRequest(http.MethodPost, url.String(), body) r.Header.Add("Content-Type", formWriter.FormDataContentType()) if err != nil { @@ -1951,8 +2014,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { r.Header.Set("Content-Type", formWriter.FormDataContentType()) r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) return r }(), @@ -1960,11 +2023,12 @@ func TestHandlers_Requiring_Signature(t *testing.T) { alloc: alloc, begin: func() { dataToEncrypt := "data_to_encrypt" - encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) if err != nil { t.Fatal(err) } - header := make([]byte, 2*1024) + + header := make([]byte, HeaderChecksumSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) @@ -2000,11 +2064,11 @@ func TestHandlers_Requiring_Signature(t *testing.T) { WithArgs(alloc.ID, filePathHash). WillReturnRows( sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/", fileref.CHUNK_SIZE), + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/", fileref.CHUNK_SIZE), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(guestClient.ClientID). WillReturnError(gorm.ErrRecordNotFound) rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/") @@ -2015,27 +2079,32 @@ func TestHandlers_Requiring_Signature(t *testing.T) { AddRow("/", "d", rootPathHash, rootPathHash, "content_hash", "", "."), ) + gpbk, err := guestScheme.GetPublicKey() + if err != nil { + t.Fatal(err) + } + + reEncryptionKey, _ := ownerScheme.GetReGenKey(gpbk, "filetype:audio") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(guestClient.ClientID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, gpbk), + ) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(guestClient.ClientID). WillReturnRows( sqlmock.NewRows([]string{"client_id"}). - AddRow(client.GetClientID()), + AddRow(guestClient.ClientID), ) aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). + WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) - reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs(client.GetClientID(), rootPathHash). - WillReturnRows( - sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). - AddRow(reEncryptionKey, encscheme.GetEncryptedKey()), - ) - mock.ExpectCommit() }, wantCode: http.StatusOK, @@ -2062,28 +2131,30 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") if err != nil { t.Fatal(err) } require.NoError(t, formWriter.WriteField("auth_token", authTicket)) rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey rm.BlobberID = "" rm.AllocationID = alloc.ID rm.ReadCounter = 1 - rm.OwnerID = client.GetClientID() - err = rm.Sign() + rm.OwnerID = alloc.OwnerID + rm.Signature, err = signHash(guestClient, rm.GetHash()) if err != nil { t.Fatal(err) } + rmData, err := json.Marshal(rm) require.NoError(t, err) require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) if err := formWriter.Close(); err != nil { t.Fatal(err) } + r, err := http.NewRequest(http.MethodPost, url.String(), body) r.Header.Add("Content-Type", formWriter.FormDataContentType()) if err != nil { @@ -2098,8 +2169,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { r.Header.Set("Content-Type", formWriter.FormDataContentType()) r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) return r }(), @@ -2107,11 +2178,12 @@ func TestHandlers_Requiring_Signature(t *testing.T) { alloc: alloc, begin: func() { dataToEncrypt := "data_to_encrypt" - encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) if err != nil { t.Fatal(err) } - header := make([]byte, 2*1024) + + header := make([]byte, HeaderChecksumSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) @@ -2147,11 +2219,11 @@ func TestHandlers_Requiring_Signature(t *testing.T) { WithArgs(alloc.ID, filePathHash). WillReturnRows( sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). - AddRow("/folder1/subfolder1/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder1/subfolder1", filestore.CHUNK_SIZE), + AddRow("/folder1/subfolder1/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/folder1/subfolder1", filestore.CHUNK_SIZE), ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(guestClient.ClientID). WillReturnError(gorm.ErrRecordNotFound) rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") @@ -2162,27 +2234,32 @@ func TestHandlers_Requiring_Signature(t *testing.T) { AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "."), ) + gpbk, err := guestScheme.GetPublicKey() + if err != nil { + t.Fatal(err) + } + + reEncryptionKey, _ := ownerScheme.GetReGenKey(gpbk, "filetype:audio") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(guestClient.ClientID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, gpbk), + ) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(client.GetClientID()). + WithArgs(guestClient.ClientID). WillReturnRows( sqlmock.NewRows([]string{"client_id"}). - AddRow(client.GetClientID()), + AddRow(guestClient.ClientID), ) aa := sqlmock.AnyArg() mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). + WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). WillReturnResult(sqlmock.NewResult(0, 0)) - reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs(client.GetClientID(), rootPathHash). - WillReturnRows( - sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). - AddRow(reEncryptionKey, encscheme.GetEncryptedKey()), - ) - mock.ExpectCommit() }, wantCode: http.StatusOK, @@ -2209,28 +2286,30 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") if err != nil { t.Fatal(err) } require.NoError(t, formWriter.WriteField("auth_token", authTicket)) rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey rm.BlobberID = "" rm.AllocationID = alloc.ID rm.ReadCounter = 1 - rm.OwnerID = client.GetClientID() - err = rm.Sign() + rm.OwnerID = alloc.OwnerID + rm.Signature, err = signHash(guestClient, rm.GetHash()) if err != nil { t.Fatal(err) } + rmData, err := json.Marshal(rm) require.NoError(t, err) require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) if err := formWriter.Close(); err != nil { t.Fatal(err) } + r, err := http.NewRequest(http.MethodPost, url.String(), body) r.Header.Add("Content-Type", formWriter.FormDataContentType()) if err != nil { @@ -2245,8 +2324,8 @@ func TestHandlers_Requiring_Signature(t *testing.T) { r.Header.Set("Content-Type", formWriter.FormDataContentType()) r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) return r }(), @@ -2254,11 +2333,12 @@ func TestHandlers_Requiring_Signature(t *testing.T) { alloc: alloc, begin: func() { dataToEncrypt := "data_to_encrypt" - encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) if err != nil { t.Fatal(err) } - header := make([]byte, 2*1024) + + header := make([]byte, HeaderChecksumSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) @@ -2294,7 +2374,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { WithArgs(alloc.ID, filePathHash). WillReturnRows( sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey(), "/folder2/subfolder1", fileref.CHUNK_SIZE), + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/folder2/subfolder1", fileref.CHUNK_SIZE), ) rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") @@ -2304,12 +2384,16 @@ func TestHandlers_Requiring_Signature(t *testing.T) { sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "/"), ) + }, wantCode: http.StatusBadRequest, wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: cannot verify auth ticket: invalid_parameters: Auth ticket is not valid for the resource being requested\"}\n\n", }, } + tests := append(positiveTests, negativeTests...) + // tests := positiveTests + for _, test := range tests { t.Run(test.name, func(t *testing.T) { mock := datastore.MockTheStore(t) @@ -2323,6 +2407,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { test.end() } + fmt.Printf("\nResponse body: %v", test.args.w.Body.String()) assert.Equal(t, test.wantCode, test.args.w.Result().StatusCode) if test.wantCode != http.StatusOK || test.wantBody != "" { assert.Equal(t, test.wantBody, test.args.w.Body.String()) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 5aca5f898..b225e1331 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -11,7 +11,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" zencryption "github.com/0chain/gosdk/zboxcore/encryption" "net/http" @@ -30,7 +29,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/gosdk/constants" - zfileref "github.com/0chain/gosdk/zboxcore/fileref" "gorm.io/datatypes" "gorm.io/gorm" @@ -39,6 +37,13 @@ import ( "go.uber.org/zap" ) +const ( + // EncryptionOverHead takes blockSize increment when data is incremented. + // messageCheckSum(128) + overallChecksum(128) + ","(1) + data-size-increment(16) + EncryptionOverHead = 273 + HeaderChecksumSize = 2048 // Change it to 257 after header size is fixed in chunked upload as well +) + func readPreRedeem(ctx context.Context, alloc *allocation.Allocation, numBlocks, pendNumBlocks int64, payerID string) (err error) { if numBlocks == 0 { return @@ -179,27 +184,21 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r var ( clientID = ctx.Value(constants.ContextKeyClient).(string) allocationTx = ctx.Value(constants.ContextKeyAllocation).(string) - _ = ctx.Value(constants.ContextKeyClientKey).(string) // runtime type check alloc *allocation.Allocation ) - // check client if clientID == "" { return nil, common.NewError("download_file", "invalid client") } - // get and check allocation alloc, err = fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { - return nil, common.NewErrorf("download_file", - "invalid allocation id passed: %v", err) + return nil, common.NewErrorf("download_file", "invalid allocation id passed: %v", err) } - // get and parse file params if err = r.ParseMultipartForm(FormFileParseMaxMemory); err != nil { Logger.Info("download_file - request_parse_error", zap.Error(err)) - return nil, common.NewErrorf("download_file", - "request_parse_error: %v", err) + return nil, common.NewErrorf("download_file", "request_parse_error: %v", err) } pathHash, err := pathHashFromReq(r, alloc.ID) @@ -214,119 +213,93 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r var blockNum int64 blockNum, err = strconv.ParseInt(blockNumStr, 10, 64) - if err != nil || blockNum < 0 { + if err != nil || blockNum < 1 { return nil, common.NewError("download_file", "invalid block number") } - var numBlocksStr = r.FormValue("num_blocks") - if numBlocksStr == "" { - numBlocksStr = "1" - } + numBlocks := int64(1) + numBlocksStr := r.FormValue("num_blocks") - var numBlocks int64 - numBlocks, err = strconv.ParseInt(numBlocksStr, 10, 64) - if err != nil || numBlocks < 0 { - return nil, common.NewError("download_file", - "invalid number of blocks") + if numBlocksStr != "" { + numBlocks, err = strconv.ParseInt(numBlocksStr, 10, 64) + if err != nil || numBlocks <= 0 { + return nil, common.NewError("download_file", "invalid number of blocks") + } } - // get read marker - var ( - readMarkerString = r.FormValue("read_marker") - readMarker = &readmarker.ReadMarker{} - ) - err = json.Unmarshal([]byte(readMarkerString), &readMarker) - if err != nil { + readMarkerString := r.FormValue("read_marker") + readMarker := new(readmarker.ReadMarker) + + if err := json.Unmarshal([]byte(readMarkerString), readMarker); err != nil { return nil, common.NewErrorf("download_file", "invalid parameters, "+ "error parsing the readmarker for download: %v", err) } - var rmObj = &readmarker.ReadMarkerEntity{} + rmObj := new(readmarker.ReadMarkerEntity) rmObj.LatestRM = readMarker if err = rmObj.VerifyMarker(ctx, alloc); err != nil { - return nil, common.NewErrorf("download_file", "invalid read marker, "+ - "failed to verify the read marker: %v", err) + return nil, common.NewErrorf("download_file", "invalid read marker, "+"failed to verify the read marker: %v", err) } // get file reference - var fileref *reference.Ref - fileref, err = reference.GetReferenceFromLookupHash(ctx, alloc.ID, pathHash) + fileref, err := reference.GetReferenceFromLookupHash(ctx, alloc.ID, pathHash) if err != nil { - return nil, common.NewErrorf("download_file", - "invalid file path: %v", err) + return nil, common.NewErrorf("download_file", "invalid file path: %v", err) } if fileref.Type != reference.FILE { - return nil, common.NewErrorf("download_file", - "path is not a file: %v", err) + return nil, common.NewErrorf("download_file", "path is not a file: %v", err) } // set payer: default - var payerID = alloc.OwnerID + payerID := alloc.OwnerID // set payer: check for explicit allocation payer value - if len(alloc.PayerID) > 0 { + if alloc.PayerID != "" { payerID = alloc.PayerID } - // authorize file access - var ( - isOwner = clientID == alloc.OwnerID - isRepairer = clientID == alloc.RepairerID - isCollaborator = reference.IsACollaborator(ctx, fileref.ID, clientID) - ) + isOwner := clientID == alloc.OwnerID + isRepairer := clientID == alloc.RepairerID + isCollaborator := reference.IsACollaborator(ctx, fileref.ID, clientID) - var authToken *readmarker.AuthTicket = nil + var authToken *readmarker.AuthTicket + var shareInfo *reference.ShareInfo - if (!isOwner && !isRepairer && !isCollaborator) || len(r.FormValue("auth_token")) > 0 { - var authTokenString = r.FormValue("auth_token") + if !(isOwner || isCollaborator) { + authTokenString := r.FormValue("auth_token") + if authTokenString == "" { + return nil, common.NewError("invalid_client", "authticket is required") + } - // check auth token - if authToken, err := fsh.verifyAuthTicket(ctx, authTokenString, alloc, fileref, clientID); authToken == nil { + if authToken, err = fsh.verifyAuthTicket(ctx, authTokenString, alloc, fileref, clientID); authToken == nil { return nil, common.NewErrorf("download_file", "cannot verify auth ticket: %v", err) } - authToken = &readmarker.AuthTicket{} - err = json.Unmarshal([]byte(authTokenString), &authToken) - if err != nil { - return nil, common.NewErrorf("download_file", - "error parsing the auth ticket for download: %v", err) + shareInfo, err = reference.GetShareInfo(ctx, authToken.ClientID, authToken.FilePathHash) + if err != nil || shareInfo == nil { + return nil, errors.New("client does not have permission to download the file. share does not exist") + } + + if shareInfo.Revoked { + return nil, errors.New("client does not have permission to download the file. share revoked") } + // set payer: check for command line payer flag (--rx_pay) if r.FormValue("rx_pay") == "true" { payerID = clientID } - // we only check content hash if its authticket is referring to a file - if authToken.RefType == zfileref.FILE && authToken.ActualFileHash != fileref.ActualFileHash { - return nil, errors.New("content hash does not match the requested file content hash") - } - - if authToken.RefType == zfileref.DIRECTORY { - hashes := util.GetParentPathHashes(allocationTx, fileref.Path) - found := false - for _, hash := range hashes { - if hash == authToken.FilePathHash { - found = true - break - } - } - if !found { - return nil, errors.New("auth ticket is not authorized to download file specified") - } - } readMarker.AuthTicket = datatypes.JSON(authTokenString) // check for file payer flag if fileAttrs, err := fileref.GetAttributes(); err != nil { - return nil, common.NewErrorf("download_file", - "error getting file attributes: %v", err) - } else if fileAttrs.WhoPaysForReads == common.WhoPays3rdParty { + return nil, common.NewErrorf("download_file", "error getting file attributes: %v", err) + } else if fileAttrs.WhoPaysForReads == common.WhoPays3rdParty && !(isCollaborator || isRepairer) { payerID = clientID } } - // create read marker var ( rme *readmarker.ReadMarkerEntity @@ -336,33 +309,29 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r rme, err = readmarker.GetLatestReadMarkerEntity(ctx, clientID) if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, common.NewErrorf("download_file", - "couldn't get read marker from DB: %v", err) + return nil, common.NewErrorf("download_file", "couldn't get read marker from DB: %v", err) } if rme != nil { latestRM = rme.LatestRM if pendNumBlocks, err = rme.PendNumBlocks(); err != nil { - return nil, common.NewErrorf("download_file", - "couldn't get number of blocks pending redeeming: %v", err) + return nil, common.NewErrorf("download_file", "couldn't get number of blocks pending redeeming: %v", err) } } if latestRM != nil && latestRM.ReadCounter+(numBlocks) != readMarker.ReadCounter { - var response = &blobberhttp.DownloadResponse{ + return &blobberhttp.DownloadResponse{ Success: false, LatestRM: latestRM, Path: fileref.Path, AllocationID: fileref.AllocationID, - } - return response, nil + }, nil } // check out read pool tokens if read_price > 0 err = readPreRedeem(ctx, alloc, numBlocks, pendNumBlocks, payerID) if err != nil { - return nil, common.NewErrorf("download_file", - "pre-redeeming read marker: %v", err) + return nil, common.NewErrorf("download_file", "pre-redeeming read marker: %v", err) } // reading is allowed @@ -370,18 +339,16 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r downloadMode = r.FormValue("content") respData []byte ) - if len(downloadMode) > 0 && downloadMode == DownloadContentThumb { + if downloadMode == DownloadContentThumb { var fileData = &filestore.FileInputData{} fileData.Name = fileref.Name fileData.Path = fileref.Path fileData.Hash = fileref.ThumbnailHash fileData.OnCloud = fileref.OnCloud fileData.ChunkSize = fileref.ChunkSize - respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, - fileData, blockNum, numBlocks) + respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, blockNum, numBlocks) if err != nil { - return nil, common.NewErrorf("download_file", - "couldn't get thumbnail block: %v", err) + return nil, common.NewErrorf("download_file", "couldn't get thumbnail block: %v", err) } } else { var fileData = &filestore.FileInputData{} @@ -391,48 +358,25 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r fileData.OnCloud = fileref.OnCloud fileData.ChunkSize = fileref.ChunkSize - respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, - fileData, blockNum, numBlocks) + respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, blockNum, numBlocks) if err != nil { - return nil, common.NewErrorf("download_file", - "couldn't get file block: %v", err) + return nil, common.NewErrorf("download_file", "couldn't get file block: %v", err) } } readMarker.PayerID = payerID err = readmarker.SaveLatestReadMarker(ctx, readMarker, latestRM == nil) if err != nil { - return nil, common.NewErrorf("download_file", - "couldn't save latest read marker: %v", err) - } - - var shareInfo *reference.ShareInfo - if authToken != nil { - shareInfo, err = reference.GetShareInfo( - ctx, - readMarker.ClientID, - authToken.FilePathHash, - ) - - if err == nil && shareInfo.Revoked { - return nil, errors.New("client does not have permission to download the file. share revoked") - } + Logger.Error(err.Error()) + return nil, common.NewErrorf("download_file", "couldn't save latest read marker") } - if len(fileref.EncryptedKey) > 0 && authToken != nil { - // should not happen, just in case - if shareInfo == nil { - return nil, errors.New("client does not have permission to download the file. share does not exist") - } - - buyerEncryptionPublicKey := shareInfo.ClientEncryptionPublicKey + if fileref.EncryptedKey != "" && authToken != nil { encscheme := zencryption.NewEncryptionScheme() - // reEncrypt does not require pub / private key, - // we could probably make it a classless function - if _, err := encscheme.Initialize(""); err != nil { return nil, err } + if err := encscheme.InitForDecryption("filetype:audio", fileref.EncryptedKey); err != nil { return nil, err } @@ -440,12 +384,12 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r totalSize := len(respData) result := []byte{} for i := 0; i < totalSize; i += int(fileref.ChunkSize) { - encMsg := &zencryption.EncryptedMessage{} + encMsg := new(zencryption.EncryptedMessage) chunkData := respData[i:int64(math.Min(float64(i+int(fileref.ChunkSize)), float64(totalSize)))] - encMsg.EncryptedData = chunkData[(2 * 1024):] + encMsg.EncryptedData = chunkData[HeaderChecksumSize:] - headerBytes := chunkData[:(2 * 1024)] + headerBytes := chunkData[:HeaderChecksumSize] headerBytes = bytes.Trim(headerBytes, "\x00") headerString := string(headerBytes) @@ -458,7 +402,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r encMsg.MessageChecksum, encMsg.OverallChecksum = headerChecksums[0], headerChecksums[1] encMsg.EncryptedKey = encscheme.GetEncryptedKey() - reEncMsg, err := encscheme.ReEncrypt(encMsg, shareInfo.ReEncryptionKey, buyerEncryptionPublicKey) + reEncMsg, err := encscheme.ReEncrypt(encMsg, shareInfo.ReEncryptionKey, shareInfo.ClientEncryptionPublicKey) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index f4dc23b8d..c98dbe92d 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -81,13 +81,14 @@ func TestDownloadFile(t *testing.T) { parameters struct { isOwner bool isCollaborator bool + isRepairer bool useAuthTicket bool - attribute common.WhoPays - payerId client.Client isRevoked bool isFundedBlobber bool isFunded0Chain bool rxPay bool + attribute common.WhoPays + payerId client.Client // client input from gosdk's BlockDownloadRequest, inData blockDownloadRequest @@ -199,18 +200,34 @@ func TestDownloadFile(t *testing.T) { p parameters, rm marker.ReadMarker, ) { - mocket.Catcher.NewMock().OneTime().WithQuery( - `SELECT * FROM "allocations" WHERE`, - ).WithArgs( - "mock_allocation_Tx", - ).OneTime().WithReply( - []map[string]interface{}{{ - "id": p.allocation.ID, - "expiration_date": mockLongTimeInFuture, - "owner_id": mockOwner.ClientID, - "owner_public_key": mockOwner.ClientKey, - }}, - ) + if p.isRepairer { + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "allocations" WHERE`, + ).WithArgs( + "mock_allocation_Tx", + ).OneTime().WithReply( + []map[string]interface{}{{ + "id": p.allocation.ID, + "expiration_date": mockLongTimeInFuture, + "owner_id": mockOwner.ClientID, + "owner_public_key": mockOwner.ClientKey, + "repairer_id": mockClient.ClientID, + }}, + ) + } else { + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "allocations" WHERE`, + ).WithArgs( + "mock_allocation_Tx", + ).OneTime().WithReply( + []map[string]interface{}{{ + "id": p.allocation.ID, + "expiration_date": mockLongTimeInFuture, + "owner_id": mockOwner.ClientID, + "owner_public_key": mockOwner.ClientKey, + }}, + ) + } mocket.Catcher.NewMock().OneTime().WithQuery( `SELECT * FROM "terms" WHERE`, @@ -436,9 +453,17 @@ func TestDownloadFile(t *testing.T) { numBlocks: 10240, rxPay: p.rxPay, } - p.allocation = allocation.Allocation{ - ID: mockAllocationId, - Tx: mockAllocationTx, + if p.isRepairer { + p.allocation = allocation.Allocation{ + ID: mockAllocationId, + Tx: mockAllocationTx, + RepairerID: mockClient.ClientID, + } + } else { + p.allocation = allocation.Allocation{ + ID: mockAllocationId, + Tx: mockAllocationTx, + } } require.True(t, (p.isOwner && !p.isCollaborator && !p.useAuthTicket) || !p.isOwner) require.True(t, p.attribute == common.WhoPays3rdParty || p.attribute == common.WhoPaysOwner) @@ -458,6 +483,7 @@ func TestDownloadFile(t *testing.T) { parameters: parameters{ isOwner: true, isCollaborator: false, + isRepairer: false, useAuthTicket: false, attribute: common.WhoPays3rdParty, isRevoked: false, @@ -471,6 +497,7 @@ func TestDownloadFile(t *testing.T) { parameters: parameters{ isOwner: true, isCollaborator: false, + isRepairer: false, useAuthTicket: false, attribute: common.WhoPays3rdParty, isRevoked: false, @@ -484,6 +511,7 @@ func TestDownloadFile(t *testing.T) { parameters: parameters{ isOwner: true, isCollaborator: false, + isRepairer: false, useAuthTicket: false, attribute: common.WhoPays3rdParty, isRevoked: false, @@ -497,10 +525,11 @@ func TestDownloadFile(t *testing.T) { }, }, { - name: "ok_collaborator", + name: "err_collaborator_without_authticket", parameters: parameters{ isOwner: false, isCollaborator: true, + isRepairer: false, useAuthTicket: false, attribute: common.WhoPays3rdParty, isRevoked: false, @@ -508,12 +537,30 @@ func TestDownloadFile(t *testing.T) { isFunded0Chain: true, rxPay: false, }, + want: want{ + err: false, + }, + }, + { + name: "ok_collaborator_with_authticket", + parameters: parameters{ + isOwner: false, + isCollaborator: true, + isRepairer: false, + useAuthTicket: true, + attribute: common.WhoPays3rdParty, + isRevoked: false, + isFundedBlobber: true, + isFunded0Chain: true, + rxPay: false, + }, }, { name: "ok_authTicket_wp_owner", parameters: parameters{ isOwner: false, isCollaborator: false, + isRepairer: false, useAuthTicket: true, attribute: common.WhoPaysOwner, isRevoked: false, @@ -527,6 +574,7 @@ func TestDownloadFile(t *testing.T) { parameters: parameters{ isOwner: false, isCollaborator: false, + isRepairer: false, useAuthTicket: true, attribute: common.WhoPays3rdParty, isRevoked: false, @@ -540,6 +588,7 @@ func TestDownloadFile(t *testing.T) { parameters: parameters{ isOwner: false, isCollaborator: false, + isRepairer: false, useAuthTicket: true, attribute: common.WhoPays3rdParty, isRevoked: true, @@ -557,6 +606,21 @@ func TestDownloadFile(t *testing.T) { parameters: parameters{ isOwner: false, isCollaborator: false, + isRepairer: false, + useAuthTicket: true, + attribute: common.WhoPaysOwner, + isRevoked: false, + isFundedBlobber: false, + isFunded0Chain: true, + rxPay: true, + }, + }, + { + name: "ok_repairer_with_authticket", + parameters: parameters{ + isOwner: false, + isCollaborator: false, + isRepairer: true, useAuthTicket: true, attribute: common.WhoPaysOwner, isRevoked: false, @@ -565,6 +629,24 @@ func TestDownloadFile(t *testing.T) { rxPay: true, }, }, + { + name: "err_repairer_without_authticket", + parameters: parameters{ + isOwner: false, + isCollaborator: false, + isRepairer: true, + useAuthTicket: false, + attribute: common.WhoPaysOwner, + isRevoked: false, + isFundedBlobber: false, + isFunded0Chain: true, + rxPay: true, + }, + want: want{ + err: true, + errMsg: "invalid_client: authticket is required", + }, + }, } for _, test := range tests { t.Run(test.name, @@ -589,6 +671,7 @@ func TestDownloadFile(t *testing.T) { require.EqualValues(t, test.want.errMsg, err.Error()) return } + }, ) } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 36415f0ae..e543f02cc 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -975,19 +975,9 @@ func pathsFromReq(r *http.Request) ([]string, error) { return paths, nil } -func pathHashFromReq(r *http.Request, allocationID string) (string, error) { - var ( - pathHash = r.FormValue("path_hash") - path = r.FormValue("path") - ) - if pathHash == "" { - if path == "" { - return "", common.NewError("invalid_parameters", "Invalid path") - } - pathHash = reference.GetReferenceLookup(allocationID, path) - } - - return pathHash, nil +func pathHashFromReq(r *http.Request, allocationID string) (pathHash string, err error) { + pathHash, _, err = getPathHash(r, allocationID) + return } func getPathHash(r *http.Request, allocationID string) (pathHash, path string, err error) { diff --git a/go.mod b/go.mod index 3cc92ef50..c351a4708 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.4.1-0.20220117131359-7dd554e5a0f5 + github.com/0chain/gosdk v1.4.7 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index ecf7ab316..bba4eb086 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.4.1-0.20220117131359-7dd554e5a0f5 h1:39ZQnIjlTe0JBIn8wrLQqMMFhbZX900qdMSveO8EUhc= -github.com/0chain/gosdk v1.4.1-0.20220117131359-7dd554e5a0f5/go.mod h1:FB2xXhQyIM1vwvQ1jC98wNclbDTBwqrG+Z/IQC0LaBs= +github.com/0chain/gosdk v1.4.7 h1:xxy80bIWthbVbgK/r+BLldEkwWwpAsFZKiziZ5GxawQ= +github.com/0chain/gosdk v1.4.7/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= From 22e61322b76f68f116ec5d4c9c0c31ec8e2ac3ee Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 11 Feb 2022 13:09:42 +0800 Subject: [PATCH 102/161] fix(merkle): fixed merkle chunk size for challenge hash (#533) * fix(merkle): fixed merkle chunk size for challenge hash * fix(gomod): upgraded zap and grpc * fix(gomod): upgraded gosdk * fix(merkel):fixed merkel chunk size bug * fix(merkel):fixed panic if chunksize is less than 1024 * fix(gomod): upgraded gosdk --- .../blobbercore/challenge/protocol.go | 7 +++++- .../blobbercore/filestore/fs_store.go | 15 +++++++++++-- go.mod | 12 +++++----- go.sum | 22 +++++++++++-------- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 8db4deefa..c8d28d8ad 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -149,10 +149,15 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { inputData.ChunkSize = objectPath.ChunkSize maxNumBlocks := 1024 + merkleChunkSize := objectPath.ChunkSize / 1024 + // chunksize is less than 1024 + if merkleChunkSize == 0 { + merkleChunkSize = 1 + } // the file is too small, some of 1024 blocks is not filled if objectPath.Size < objectPath.ChunkSize { - merkleChunkSize := objectPath.ChunkSize / 1024 + maxNumBlocks = int(math.Ceil(float64(objectPath.Size) / float64(merkleChunkSize))) } diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index f68812872..fd1e7c9f9 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -333,17 +333,28 @@ func (fs *FileFSStore) GetFileBlockForChallenge(allocationID string, fileData *F return nil, nil, errors.ThrowLog(err2.Error(), constants.ErrUnableHash) } - merkleChunkSize := int(fileData.ChunkSize / 1024) + merkleChunkSize := int(fileData.ChunkSize) / 1024 + + if merkleChunkSize == 0 { + merkleChunkSize = 1 + } + + offset := 0 + for i := 0; i < len(dataBytes); i += merkleChunkSize { end := i + merkleChunkSize if end > len(dataBytes) { end = len(dataBytes) } - offset := i / merkleChunkSize if offset == blockoffset { returnBytes = append(returnBytes, dataBytes[i:end]...) } + + offset++ + if offset >= 1024 { + offset = 1 + } } bytesBuf.Reset() } diff --git a/go.mod b/go.mod index c351a4708..58784a021 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.4.7 + github.com/0chain/gosdk v1.5.1-0.20220211020847-2b54b2fa404f github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.1 github.com/go-ini/ini v1.55.0 // indirect @@ -25,12 +25,12 @@ require ( github.com/spf13/viper v1.10.1 github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 - go.uber.org/zap v1.20.0 + go.uber.org/zap v1.21.0 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 - golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect - google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 - google.golang.org/grpc v1.43.0 + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068 + google.golang.org/grpc v1.44.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 google.golang.org/protobuf v1.27.1 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect diff --git a/go.sum b/go.sum index bba4eb086..ace573d4c 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.4.7 h1:xxy80bIWthbVbgK/r+BLldEkwWwpAsFZKiziZ5GxawQ= -github.com/0chain/gosdk v1.4.7/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= +github.com/0chain/gosdk v1.5.1-0.20220211020847-2b54b2fa404f h1:dLjkhdiMIY9Rs3mUlEAPqAQKf+lE1NLKFd+bzVkQyzk= +github.com/0chain/gosdk v1.5.1-0.20220211020847-2b54b2fa404f/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -1049,8 +1049,8 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= -go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1182,8 +1182,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1308,10 +1308,12 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1521,8 +1523,9 @@ google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 h1:zzNejm+EgrbLfDZ6lu9Uud2IVvHySPl8vQzf04laR5Q= google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068 h1:pwzFiZfBTH/GjBWz1BcDwMBaHBo8mZvpLa7eBKJpFAk= +google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1556,8 +1559,9 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= From a2465dda83fe555834cda8ce7c8e9e778963a431 Mon Sep 17 00:00:00 2001 From: Lz Date: Mon, 14 Feb 2022 07:25:40 +0800 Subject: [PATCH 103/161] fix(download): fixed encryption header issues (#536) * fix(download): fixed encryption header issues * fix(download): fixed encryption header issues --- .../blobbercore/handler/chunk_encoder.go | 88 +++++++++++++++++++ .../blobbercore/handler/handler_test.go | 8 +- .../handler/object_operation_handler.go | 66 ++++---------- 3 files changed, 109 insertions(+), 53 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/chunk_encoder.go diff --git a/code/go/0chain.net/blobbercore/handler/chunk_encoder.go b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go new file mode 100644 index 000000000..18e954ef0 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go @@ -0,0 +1,88 @@ +package handler + +import ( + "bytes" + "errors" + "strings" + + zencryption "github.com/0chain/gosdk/zboxcore/encryption" +) + +// ChunkEncoder encode/decode chunk data +type ChunkEncoder interface { + // Encode encode chunk data if it is necessary + Encode(chunkSize int, data []byte) ([]byte, error) +} + +// RawChunkEncoder raw chunk data +type RawChunkEncoder struct { +} + +// Encode read chunk data +func (r *RawChunkEncoder) Encode(chunkSize int, data []byte) ([]byte, error) { + return data, nil +} + +// PREChunkEncoder encode and decode chunk data with PREEncryptionScheme if file is shared with auth token +type PREChunkEncoder struct { + EncryptedKey string + ReEncryptionKey string + ClientEncryptionPublicKey string +} + +// Encode encode chunk data with PREEncryptionScheme for subscriber to download it +func (r *PREChunkEncoder) Encode(chunkSize int, data []byte) ([]byte, error) { + encscheme := zencryption.NewEncryptionScheme() + if _, err := encscheme.Initialize(""); err != nil { + return nil, err + } + + if err := encscheme.InitForDecryption("filetype:audio", r.EncryptedKey); err != nil { + return nil, err + } + + totalSize := len(data) + result := []byte{} + + for i := 0; i < totalSize; i += chunkSize { + encMsg := new(zencryption.EncryptedMessage) + + nextIndex := i + chunkSize + var chunkData []byte + if nextIndex > totalSize { + chunkData = make([]byte, totalSize-i) + nextIndex = totalSize + } else { + chunkData = make([]byte, chunkSize) + } + + copy(chunkData, data[i:nextIndex]) + + encMsg.EncryptedData = chunkData[EncryptionHeaderSize:] + + headerBytes := chunkData[:EncryptionHeaderSize] + headerBytes = bytes.Trim(headerBytes, "\x00") + headerString := string(headerBytes) + + headerChecksums := strings.Split(headerString, ",") + if len(headerChecksums) != 2 { + return nil, errors.New("Block has invalid encryption header") + } + + encMsg.MessageChecksum, encMsg.OverallChecksum = headerChecksums[0], headerChecksums[1] + encMsg.EncryptedKey = encscheme.GetEncryptedKey() + + reEncMsg, err := encscheme.ReEncrypt(encMsg, r.ReEncryptionKey, r.ClientEncryptionPublicKey) + if err != nil { + return nil, err + } + + encData, err := reEncMsg.Marshal() + if err != nil { + return nil, err + } + // 256 bytes to save ReEncryption header instead of 2048 EncryptionHeader + result = append(result, encData...) + } + return result, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index a960b0a32..92e15879c 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -1876,7 +1876,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } - header := make([]byte, HeaderChecksumSize) + header := make([]byte, EncryptionHeaderSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) fmt.Println("Encrypted data: ", string(data)) @@ -2028,7 +2028,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } - header := make([]byte, HeaderChecksumSize) + header := make([]byte, EncryptionHeaderSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) @@ -2183,7 +2183,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } - header := make([]byte, HeaderChecksumSize) + header := make([]byte, EncryptionHeaderSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) @@ -2338,7 +2338,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Fatal(err) } - header := make([]byte, HeaderChecksumSize) + header := make([]byte, EncryptionHeaderSize) copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index b225e1331..9053418be 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1,17 +1,13 @@ package handler import ( - "bytes" "context" "encoding/hex" "encoding/json" "errors" - "math" - "strings" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" - zencryption "github.com/0chain/gosdk/zboxcore/encryption" "net/http" "path/filepath" @@ -38,10 +34,10 @@ import ( ) const ( - // EncryptionOverHead takes blockSize increment when data is incremented. - // messageCheckSum(128) + overallChecksum(128) + ","(1) + data-size-increment(16) - EncryptionOverHead = 273 - HeaderChecksumSize = 2048 // Change it to 257 after header size is fixed in chunked upload as well + // EncryptionHeaderSize encryption header size in chunk + EncryptionHeaderSize = 2 * 1024 + // ReEncryptionHeaderSize re-encryption header size in chunk + ReEncryptionHeaderSize = 256 ) func readPreRedeem(ctx context.Context, alloc *allocation.Allocation, numBlocks, pendNumBlocks int64, payerID string) (err error) { @@ -371,53 +367,25 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r return nil, common.NewErrorf("download_file", "couldn't save latest read marker") } - if fileref.EncryptedKey != "" && authToken != nil { - encscheme := zencryption.NewEncryptionScheme() - if _, err := encscheme.Initialize(""); err != nil { - return nil, err + var chunkEncoder ChunkEncoder + if len(fileref.EncryptedKey) > 0 && authToken != nil { + chunkEncoder = &PREChunkEncoder{ + EncryptedKey: fileref.EncryptedKey, + ReEncryptionKey: shareInfo.ReEncryptionKey, + ClientEncryptionPublicKey: shareInfo.ClientEncryptionPublicKey, } + } else { + chunkEncoder = &RawChunkEncoder{} + } - if err := encscheme.InitForDecryption("filetype:audio", fileref.EncryptedKey); err != nil { - return nil, err - } + chunkData, err := chunkEncoder.Encode(int(fileref.ChunkSize), respData) - totalSize := len(respData) - result := []byte{} - for i := 0; i < totalSize; i += int(fileref.ChunkSize) { - encMsg := new(zencryption.EncryptedMessage) - chunkData := respData[i:int64(math.Min(float64(i+int(fileref.ChunkSize)), float64(totalSize)))] - - encMsg.EncryptedData = chunkData[HeaderChecksumSize:] - - headerBytes := chunkData[:HeaderChecksumSize] - headerBytes = bytes.Trim(headerBytes, "\x00") - headerString := string(headerBytes) - - headerChecksums := strings.Split(headerString, ",") - if len(headerChecksums) != 2 { - Logger.Error("Block has invalid header", zap.String("request Url", r.URL.String())) - return nil, errors.New("Block has invalid header for request " + r.URL.String()) - } - - encMsg.MessageChecksum, encMsg.OverallChecksum = headerChecksums[0], headerChecksums[1] - encMsg.EncryptedKey = encscheme.GetEncryptedKey() - - reEncMsg, err := encscheme.ReEncrypt(encMsg, shareInfo.ReEncryptionKey, shareInfo.ClientEncryptionPublicKey) - if err != nil { - return nil, err - } - - encData, err := reEncMsg.Marshal() - if err != nil { - return nil, err - } - result = append(result, encData...) - } - respData = result + if err != nil { + return nil, err } stats.FileBlockDownloaded(ctx, fileref.ID) - return respData, nil + return chunkData, nil } func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*blobberhttp.CommitResult, error) { From f76a6f3e0c97668b6823eaf6aad0b1fc8833a482 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 15 Feb 2022 09:22:29 +0800 Subject: [PATCH 104/161] Fix/reduce encryption header size (#538) * fix(download): reduced EncryptionHeaderSize with 257 * fix(download): reduced EncryptionHeaderSize with 256 --- code/go/0chain.net/blobbercore/handler/chunk_encoder.go | 7 ++----- code/go/0chain.net/blobbercore/handler/handler_test.go | 8 ++++---- .../blobbercore/handler/object_operation_handler.go | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/chunk_encoder.go b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go index 18e954ef0..6665b9dac 100644 --- a/code/go/0chain.net/blobbercore/handler/chunk_encoder.go +++ b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go @@ -3,7 +3,6 @@ package handler import ( "bytes" "errors" - "strings" zencryption "github.com/0chain/gosdk/zboxcore/encryption" ) @@ -62,14 +61,12 @@ func (r *PREChunkEncoder) Encode(chunkSize int, data []byte) ([]byte, error) { headerBytes := chunkData[:EncryptionHeaderSize] headerBytes = bytes.Trim(headerBytes, "\x00") - headerString := string(headerBytes) - headerChecksums := strings.Split(headerString, ",") - if len(headerChecksums) != 2 { + if len(headerBytes) != EncryptionHeaderSize { return nil, errors.New("Block has invalid encryption header") } - encMsg.MessageChecksum, encMsg.OverallChecksum = headerChecksums[0], headerChecksums[1] + encMsg.MessageChecksum, encMsg.OverallChecksum = string(headerBytes[:128]), string(headerBytes[128:]) encMsg.EncryptedKey = encscheme.GetEncryptedKey() reEncMsg, err := encscheme.ReEncrypt(encMsg, r.ReEncryptionKey, r.ClientEncryptionPublicKey) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 92e15879c..49615e78e 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -1877,7 +1877,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { } header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) fmt.Println("Encrypted data: ", string(data)) setMockFileBlock(data) @@ -2029,7 +2029,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { } header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) }, @@ -2184,7 +2184,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { } header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) }, @@ -2339,7 +2339,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { } header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+","+encMsg.OverallChecksum) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) data := append(header, encMsg.EncryptedData...) setMockFileBlock(data) }, diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 9053418be..30d17f85d 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -34,8 +34,8 @@ import ( ) const ( - // EncryptionHeaderSize encryption header size in chunk - EncryptionHeaderSize = 2 * 1024 + // EncryptionHeaderSize encryption header size in chunk: PRE.MessageChecksum(128)"+PRE.OverallChecksum(128) + EncryptionHeaderSize = 128 + 128 // ReEncryptionHeaderSize re-encryption header size in chunk ReEncryptionHeaderSize = 256 ) From 4a9422af424718fa40b0bc7c9a7b541d2cb594cd Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 17 Feb 2022 09:06:16 +0800 Subject: [PATCH 105/161] Feat/db auto migrate (#539) * feat(datastore): added AutoMigrate --- code/go/0chain.net/blobber/config.go | 2 +- code/go/0chain.net/blobber/datastore.go | 14 +++- code/go/0chain.net/blobber/filestore.go | 2 +- code/go/0chain.net/blobber/flags.go | 2 +- code/go/0chain.net/blobber/grpc.go | 3 +- code/go/0chain.net/blobber/http.go | 2 +- code/go/0chain.net/blobber/logging.go | 2 +- code/go/0chain.net/blobber/minio.go | 2 +- code/go/0chain.net/blobber/node.go | 2 +- code/go/0chain.net/blobber/zcn.go | 2 +- .../blobbercore/datastore/README.md | 43 ++++++++++++ .../blobbercore/datastore/mocket.go | 4 ++ .../blobbercore/datastore/postgres.go | 65 ++++++++++++++++++- .../datastore/postgres_migration.go | 45 +++++++++++++ .../datastore/postgres_releases.go | 12 ++++ .../blobbercore/datastore/sqlmock.go | 4 ++ .../0chain.net/blobbercore/datastore/store.go | 1 + sql/00-create-user.sql | 3 +- 18 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/datastore/README.md create mode 100644 code/go/0chain.net/blobbercore/datastore/postgres_migration.go create mode 100644 code/go/0chain.net/blobbercore/datastore/postgres_releases.go diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go index 0b4665494..2ac2c85ef 100644 --- a/code/go/0chain.net/blobber/config.go +++ b/code/go/0chain.net/blobber/config.go @@ -11,7 +11,7 @@ import ( ) func setupConfig() { - fmt.Print("[2/11] load config") + fmt.Print("[2/12] load config") // setup default config.SetupDefaultConfig() diff --git a/code/go/0chain.net/blobber/datastore.go b/code/go/0chain.net/blobber/datastore.go index acb0e0506..d0abc4843 100644 --- a/code/go/0chain.net/blobber/datastore.go +++ b/code/go/0chain.net/blobber/datastore.go @@ -9,11 +9,11 @@ import ( ) func setupDatabase() error { - fmt.Print("\r[7/11] connect data store") + fmt.Print("\r[7/12] connect data store") // check for database connection for i := 0; i < 600; i++ { if i > 0 { - fmt.Printf("\r[7/10] connect(%v) data store", i) + fmt.Printf("\r[7/12] connect(%v) data store", i) } if err := datastore.GetStore().Open(); err == nil { @@ -22,11 +22,19 @@ func setupDatabase() error { return err } fmt.Print(" [OK]\n") - return nil + break } time.Sleep(1 * time.Second) } + fmt.Println("\r[8/12] auto migrate datastore") + err := datastore.GetStore().AutoMigrate() + if err != nil { + logging.Logger.Error("Failed to migrate to the database.") + return err + } + //fmt.Print(" [OK]\n") + return nil } diff --git a/code/go/0chain.net/blobber/filestore.go b/code/go/0chain.net/blobber/filestore.go index 7c111af93..06b4a4eba 100644 --- a/code/go/0chain.net/blobber/filestore.go +++ b/code/go/0chain.net/blobber/filestore.go @@ -9,7 +9,7 @@ import ( var fsStore filestore.FileStore //nolint:unused // global which might be needed somewhere func setupFileStore() (err error) { - fmt.Print("[8/11] setup file store") + fmt.Print("[9/12] setup file store") fsStore, err = filestore.SetupFSStore(filesDir + "/files") diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go index f41af64e9..838a6c5fa 100644 --- a/code/go/0chain.net/blobber/flags.go +++ b/code/go/0chain.net/blobber/flags.go @@ -44,7 +44,7 @@ func init() { } func parseFlags() { - fmt.Print("[1/11] load flags") + fmt.Print("[1/12] load flags") flag.Parse() if filesDir == "" { diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go index cdb6ea692..d2aa9a1de 100644 --- a/code/go/0chain.net/blobber/grpc.go +++ b/code/go/0chain.net/blobber/grpc.go @@ -1,3 +1,4 @@ +//go:build !integration_tests // +build !integration_tests package main @@ -38,7 +39,7 @@ func startGRPCServer() { panic(err) } - fmt.Println("[10/11] starting grpc server [OK]") + fmt.Println("[11/12] starting grpc server [OK]") go func() { log.Fatal(grpcServer.Serve(lis)) }() diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index a946ebff9..398e832a9 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -38,7 +38,7 @@ func startHttpServer() { go startServer(&wg, r, mode, httpsPort, true) logging.Logger.Info("Ready to listen to the requests") - fmt.Println("[11/11] start http server [OK]") + fmt.Println("[12/12] start http server [OK]") wg.Wait() } diff --git a/code/go/0chain.net/blobber/logging.go b/code/go/0chain.net/blobber/logging.go index 33c449213..ccfa8a3cb 100644 --- a/code/go/0chain.net/blobber/logging.go +++ b/code/go/0chain.net/blobber/logging.go @@ -9,7 +9,7 @@ import ( ) func setupLogging() { - fmt.Print("[3/11] init logging") + fmt.Print("[3/12] init logging") if config.Development() { logging.InitLogging("development", logDir, "0chainBlobber.log") diff --git a/code/go/0chain.net/blobber/minio.go b/code/go/0chain.net/blobber/minio.go index 61d65b482..1d5140fa1 100644 --- a/code/go/0chain.net/blobber/minio.go +++ b/code/go/0chain.net/blobber/minio.go @@ -10,7 +10,7 @@ import ( ) func setupMinio() error { - fmt.Print("[4/11] setup minio") + fmt.Print("[4/12] setup minio") reader, err := os.Open(minioFile) if err != nil { return err diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index 072e47241..8fd1ee212 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -12,7 +12,7 @@ import ( ) func setupNode() error { - fmt.Print("[5/11] setup blobber") + fmt.Print("[5/12] setup blobber") reader, err := os.Open(keysFile) if err != nil { diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index 896635ee0..ab07abf98 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -17,7 +17,7 @@ import ( func setupOnChain() { //wait http & grpc startup, and go to setup on chain time.Sleep(1 * time.Second) - fmt.Println("[9/11] connecting to chain ") + fmt.Println("[10/12] connecting to chain ") const ATTEMPT_DELAY = 60 * 1 diff --git a/code/go/0chain.net/blobbercore/datastore/README.md b/code/go/0chain.net/blobbercore/datastore/README.md new file mode 100644 index 000000000..7b415c7b9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/README.md @@ -0,0 +1,43 @@ +# How to auto migrate database schema + +## what is `version`. how it works +Given a version number TABLE.INDEX.COLUMN, increment the: + +- TABLE version when you add/drop any table, +- INDEX version when you add/drop/update and index +- COLUMN version when you add/drop/update any column + +NB: current schema that is created by sql scripts is versioned as `0.0.0`. + +## How to add a new version + +### Migrate table/column in gorm.AutoMigrate + if migration works with gorm.AutoMigrate, please use it to migrate. + - update your model + - added your model in [AutoMigrate](postgres.go#L76)if it doesn't exists + ``` + db.AutoMigrate(&Migration{},&YourModel{}) + ``` + +### Migrate index/constraints manually if it is not easy to do in `AutoMigrate` +- create a new `Migration` with scripts +- append it in [releases](postgres_releases.go#L4) +``` +var releases = []Migration{ + { + Version: "0.1.0", + CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC), + Scripts: []string{ + "CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path);", + }, + }, + { + Version: "0.1.1", + CreatedAt: time.Date(2021, 10, 16, 0, 0, 0, 0, time.UTC), + Scripts: []string{ + "sql1", + "sql2", + }, + }, +} +``` \ No newline at end of file diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go index 3e8f9be7a..be6c115b1 100644 --- a/code/go/0chain.net/blobbercore/datastore/mocket.go +++ b/code/go/0chain.net/blobbercore/datastore/mocket.go @@ -84,3 +84,7 @@ func (store *Mocket) GetTransaction(ctx context.Context) *gorm.DB { func (store *Mocket) GetDB() *gorm.DB { return store.db } + +func (store *Mocket) AutoMigrate() error { + return nil +} diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index 98a0db1f0..71ec3b440 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -7,7 +7,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/core/common" - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/driver/postgres" "gorm.io/gorm" ) @@ -22,7 +23,10 @@ func (store *postgresStore) Open() error { "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", config.Configuration.DBHost, config.Configuration.DBPort, config.Configuration.DBUserName, config.Configuration.DBName, - config.Configuration.DBPassword)), &gorm.Config{}) + config.Configuration.DBPassword)), &gorm.Config{ + SkipDefaultTransaction: true, // https://gorm.io/docs/performance.html#Disable-Default-Transaction + PrepareStmt: true, //https://gorm.io/docs/performance.html#Caches-Prepared-Statement + }) if err != nil { return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) } @@ -59,10 +63,65 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *gorm.DB { if conn != nil { return conn.(*gorm.DB) } - Logger.Error("No connection in the context.") + logging.Logger.Error("No connection in the context.") return nil } func (store *postgresStore) GetDB() *gorm.DB { return store.db } + +func (store *postgresStore) AutoMigrate() error { + + err := store.db.AutoMigrate(&Migration{}) + if err != nil { + logging.Logger.Error("[db]", zap.Error(err)) + } + + if len(releases) == 0 { + fmt.Print(" + No releases [SKIP]\n") + return nil + } + + for i := 0; i < len(releases); i++ { + v := releases[i] + fmt.Print("\r + ", v.Version, " ") + isMigrated, err := store.IsMigrated(v) + if err != nil { + return err + } + if isMigrated { + fmt.Print(" [SKIP]\n") + continue + } + + err = v.Migrate(store.db) + if err != nil { + logging.Logger.Error("[db]"+v.Version, zap.Error(err)) + return err + } + + logging.Logger.Info("[db]" + v.Version + " migrated") + fmt.Print(" [OK]\n") + + } + return nil +} + +func (store *postgresStore) IsMigrated(m Migration) (bool, error) { + var version string + err := store.db. + Raw(`select version from "migrations" where version=?`, m.Version). + Pluck("version", &version). + Error + + if err == nil { + return false, nil + } + + if err == gorm.ErrRecordNotFound { + return false, nil + } + + return false, err +} diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_migration.go b/code/go/0chain.net/blobbercore/datastore/postgres_migration.go new file mode 100644 index 000000000..153fd401c --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/postgres_migration.go @@ -0,0 +1,45 @@ +package datastore + +import ( + "time" + + "gorm.io/gorm" +) + +const ( + TableNameMigration = "migrations" +) + +// Migration migration history +type Migration struct { + // Version format is "[table].[index].[column]" + // + increase table version if any table is added or deleted + // + increase index version if any index is changed + // + increase column version if any column/constraint is changed + Version string `gorm:"column:version;primary_key"` + CreatedAt time.Time `gorm:"column:created_at"` + Scripts []string `gorm:"-"` +} + +// Migrate migrate database +func (m *Migration) Migrate(db *gorm.DB) error { + return db.Transaction(func(tx *gorm.DB) error { + + for _, s := range m.Scripts { + if err := tx.Exec(s).Error; err != nil { + return err + } + } + + if err := tx.Create(m).Error; err != nil { + return err + } + + return nil + }) +} + +// TableName get table name of migrate +func (Migration) TableName() string { + return TableNameMigration +} diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_releases.go b/code/go/0chain.net/blobbercore/datastore/postgres_releases.go new file mode 100644 index 000000000..e6b83695d --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/postgres_releases.go @@ -0,0 +1,12 @@ +package datastore + +// releases migration histories +var releases = []Migration{ + // { + // Version: "0.1.0", + // CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC), + // Scripts: []string{ + // "CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path,deleted_at);", + // }, + // }, +} diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go index 1dc998035..4a0f8d446 100644 --- a/code/go/0chain.net/blobbercore/datastore/sqlmock.go +++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go @@ -79,3 +79,7 @@ func (store *Sqlmock) GetTransaction(ctx context.Context) *gorm.DB { func (store *Sqlmock) GetDB() *gorm.DB { return store.db } + +func (store *Sqlmock) AutoMigrate() error { + return nil +} diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index bfaae071d..2cf8189e5 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -24,6 +24,7 @@ type Store interface { Open() error Close() + AutoMigrate() error } var instance Store diff --git a/sql/00-create-user.sql b/sql/00-create-user.sql index 50d7988a8..a6696c1a4 100644 --- a/sql/00-create-user.sql +++ b/sql/00-create-user.sql @@ -3,4 +3,5 @@ CREATE DATABASE blobber_meta; \connect blobber_meta; CREATE USER blobber_user WITH ENCRYPTED PASSWORD 'blobber'; -GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user; \ No newline at end of file +GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user; +grant postgres to blobber_user; \ No newline at end of file From 372e64bcf53290f75d7cd3e0737bbccc107a6ed3 Mon Sep 17 00:00:00 2001 From: prince Date: Thu, 17 Feb 2022 14:15:38 +0530 Subject: [PATCH 106/161] download migrated from post to get (#540) --- code/go/0chain.net/blobbercore/handler/handler.go | 2 +- .../blobbercore/handler/object_operation_handler_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index fa76a2fa1..f10dbb5a2 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -44,7 +44,7 @@ func SetupHandlers(r *mux.Router) { //object operations r.HandleFunc("/v1/file/upload/{allocation}", common.ToJSONResponse(WithConnection(UploadHandler))) - r.HandleFunc("/v1/file/download/{allocation}", common.ToByteStream(WithConnection(DownloadHandler))).Methods(http.MethodPost, http.MethodOptions) + r.HandleFunc("/v1/file/download/{allocation}", common.ToByteStream(WithConnection(DownloadHandler))).Methods(http.MethodGet, http.MethodOptions) r.HandleFunc("/v1/file/rename/{allocation}", common.ToJSONResponse(WithConnection(RenameHandler))).Methods(http.MethodPost, http.MethodOptions) r.HandleFunc("/v1/file/copy/{allocation}", common.ToJSONResponse(WithConnection(CopyHandler))) r.HandleFunc("/v1/file/attributes/{allocation}", common.ToJSONResponse(WithConnection(UpdateAttributesHandler))) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index c98dbe92d..71dc038c7 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -435,7 +435,7 @@ func TestDownloadFile(t *testing.T) { body := new(bytes.Buffer) formWriter := multipart.NewWriter(body) rm := addToForm(t, formWriter, p) - req := httptest.NewRequest(http.MethodPost, "/v1/file/download/", body) + req := httptest.NewRequest(http.MethodGet, "/v1/file/download/", body) require.NoError(t, req.ParseForm()) req.Header.Add("Content-Type", formWriter.FormDataContentType()) return req, rm From 2479a8c2b0d4e0be4ecb67f3018697e491f1b538 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 22 Feb 2022 11:48:38 +0800 Subject: [PATCH 107/161] fix(writemarker): added distributed locks for WM (#546) --- code/go/0chain.net/blobber/config.go | 2 + .../0chain.net/blobbercore/allocation/dao.go | 53 +++++ .../blobbercore/allocation/entity.go | 12 +- .../0chain.net/blobbercore/allocation/zcn.go | 87 ++++++++ .../0chain.net/blobbercore/config/config.go | 3 + .../blobbercore/datastore/postgres.go | 2 +- .../blobbercore/datastore/postgres_schema.go | 19 ++ .../0chain.net/blobbercore/handler/context.go | 171 +++++++++++++++ .../0chain.net/blobbercore/handler/handler.go | 5 + .../handler/handler_writemarker.go | 35 +++ .../handler/handler_writemarker_test.go | 92 ++++++++ .../0chain.net/blobbercore/reference/dao.go | 73 +++++++ .../blobbercore/reference/entity.go | 69 ++++++ .../blobbercore/writemarker/mutex.go | 164 +++++++++++++++ .../blobbercore/writemarker/mutext_test.go | 199 ++++++++++++++++++ code/go/0chain.net/core/common/constants.go | 35 +++ go.mod | 13 +- go.sum | 37 ++-- 18 files changed, 1044 insertions(+), 27 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/allocation/dao.go create mode 100644 code/go/0chain.net/blobbercore/allocation/zcn.go create mode 100644 code/go/0chain.net/blobbercore/datastore/postgres_schema.go create mode 100644 code/go/0chain.net/blobbercore/handler/context.go create mode 100644 code/go/0chain.net/blobbercore/handler/handler_writemarker.go create mode 100644 code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go create mode 100644 code/go/0chain.net/blobbercore/reference/dao.go create mode 100644 code/go/0chain.net/blobbercore/reference/entity.go create mode 100644 code/go/0chain.net/blobbercore/writemarker/mutex.go create mode 100644 code/go/0chain.net/blobbercore/writemarker/mutext_test.go create mode 100644 code/go/0chain.net/core/common/constants.go diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go index 2ac2c85ef..3afef74b2 100644 --- a/code/go/0chain.net/blobber/config.go +++ b/code/go/0chain.net/blobber/config.go @@ -73,6 +73,8 @@ func setupConfig() { viper.GetDuration("write_lock_timeout") / time.Second, ) + config.Configuration.WriteMarkerLockTimeout = viper.GetDuration("write_marker_lock_timeout") + config.Configuration.UpdateAllocationsInterval = viper.GetDuration("update_allocations_interval") diff --git a/code/go/0chain.net/blobbercore/allocation/dao.go b/code/go/0chain.net/blobbercore/allocation/dao.go new file mode 100644 index 000000000..cc11aa989 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/dao.go @@ -0,0 +1,53 @@ +package allocation + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/errors" + "github.com/0chain/gosdk/constants" + "gorm.io/gorm" +) + +// GetOrCreate, get allocation if it exists in db. if not, try to sync it from blockchain, and insert it in db. +func GetOrCreate(ctx context.Context, store datastore.Store, allocationTx string) (*Allocation, error) { + db := store.GetDB() + + if len(allocationTx) == 0 { + return nil, errors.Throw(constants.ErrInvalidParameter, "tx") + } + + alloc := &Allocation{} + result := db.Table(TableNameAllocation).Where(SQLWhereGetByTx, allocationTx).First(alloc) + + if result.Error == nil { + return alloc, nil + } + + if !errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, errors.ThrowLog(result.Error.Error(), common.ErrBadDataStore) + } + + return SyncAllocation(allocationTx) + +} + +const ( + SQLWhereGetByTx = "allocations.tx = ?" +) + +// DryRun Creates a prepared statement when executing any SQL and caches them to speed up future calls +// https://gorm.io/docs/performance.html#Caches-Prepared-Statement +func DryRun(db *gorm.DB) { + + // https://gorm.io/docs/session.html#DryRun + // Session mode + tx := db.Session(&gorm.Session{PrepareStmt: true, DryRun: true}) + + // use Table instead of Model to reduce reflect times + + // prepare statement for GetOrCreate + tx.Table(TableNameAllocation).Where(SQLWhereGetByTx, "tx").First(&Allocation{}) + +} diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 0bf78f663..12b8794c1 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -18,6 +18,10 @@ const ( CHUNK_SIZE = 64 * KB ) +const ( + TableNameAllocation = "allocations" +) + type Allocation struct { ID string `gorm:"column:id;primary_key"` Tx string `gorm:"column:tx"` @@ -44,7 +48,7 @@ type Allocation struct { } func (Allocation) TableName() string { - return "allocations" + return TableNameAllocation } // RestDurationInTimeUnits returns number (float point) of time units until @@ -196,6 +200,10 @@ func (p *Pending) Save(tx *gorm.DB) error { return tx.Save(p).Error } +const ( + TableNameTerms = "terms" +) + // Terms for allocation by its Tx. type Terms struct { ID int64 `gorm:"column:id;primary_key"` @@ -207,7 +215,7 @@ type Terms struct { } func (*Terms) TableName() string { - return "terms" + return TableNameTerms } type ReadPool struct { diff --git a/code/go/0chain.net/blobbercore/allocation/zcn.go b/code/go/0chain.net/blobbercore/allocation/zcn.go new file mode 100644 index 000000000..bf7eb3778 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/zcn.go @@ -0,0 +1,87 @@ +package allocation + +import ( + "encoding/json" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "github.com/0chain/errors" + "gorm.io/gorm" +) + +// SyncAllocation try to pull allocation from blockchain, and insert it in db. +func SyncAllocation(allocationTx string) (*Allocation, error) { + t, err := transaction.VerifyTransaction(allocationTx, chain.GetServerChain()) + if err != nil { + return nil, errors.Throw(common.ErrBadRequest, + "Invalid Allocation id. Allocation not found in blockchain.") + } + var sa transaction.StorageAllocation + err = json.Unmarshal([]byte(t.TransactionOutput), &sa) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrInternal, "Error decoding the allocation transaction output.") + } + + alloc := &Allocation{} + + belongToThisBlobber := false + for _, blobberConnection := range sa.Blobbers { + if blobberConnection.ID == node.Self.ID { + belongToThisBlobber = true + + alloc.AllocationRoot = "" + alloc.BlobberSize = (sa.Size + int64(len(sa.Blobbers)-1)) / + int64(len(sa.Blobbers)) + alloc.BlobberSizeUsed = 0 + + break + } + } + if !belongToThisBlobber { + return nil, errors.Throw(common.ErrBadRequest, + "Blobber is not part of the open connection transaction") + } + + // set/update fields + alloc.ID = sa.ID + alloc.Tx = sa.Tx + alloc.Expiration = sa.Expiration + alloc.OwnerID = sa.OwnerID + alloc.OwnerPublicKey = sa.OwnerPublicKey + alloc.RepairerID = t.ClientID // blobber node id + alloc.TotalSize = sa.Size + alloc.UsedSize = sa.UsedSize + alloc.Finalized = sa.Finalized + alloc.TimeUnit = sa.TimeUnit + alloc.IsImmutable = sa.IsImmutable + + // related terms + terms := make([]*Terms, 0, len(sa.BlobberDetails)) + for _, d := range sa.BlobberDetails { + terms = append(terms, &Terms{ + BlobberID: d.BlobberID, + AllocationID: alloc.ID, + ReadPrice: d.Terms.ReadPrice, + WritePrice: d.Terms.WritePrice, + }) + } + + err = datastore.GetStore().GetDB().Transaction(func(tx *gorm.DB) error { + if err := tx.Table(TableNameAllocation).Create(alloc).Error; err != nil { + return err + } + + for _, term := range terms { + if err := tx.Table(TableNameTerms).Create(term).Error; err != nil { + return err + } + } + + return nil + }) + + return alloc, err +} diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index c574c44eb..0d12680d4 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -33,6 +33,7 @@ func SetupDefaultConfig() { viper.SetDefault("challenge_completion_time", time.Duration(-1)) viper.SetDefault("read_lock_timeout", time.Duration(-1)) viper.SetDefault("write_lock_timeout", time.Duration(-1)) + viper.SetDefault("write_marker_lock_timeout", time.Second*30) viper.SetDefault("delegate_wallet", "") viper.SetDefault("min_stake", 1.0) @@ -115,6 +116,8 @@ type Config struct { ReadLockTimeout int64 // seconds WriteLockTimeout int64 // seconds + // WriteMarkerLockTimeout lock is released automatically if it is timeout + WriteMarkerLockTimeout time.Duration UpdateAllocationsInterval time.Duration diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index 71ec3b440..b622a3ba4 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -73,7 +73,7 @@ func (store *postgresStore) GetDB() *gorm.DB { func (store *postgresStore) AutoMigrate() error { - err := store.db.AutoMigrate(&Migration{}) + err := store.db.AutoMigrate(&Migration{}, &WriteLock{}) if err != nil { logging.Logger.Error("[db]", zap.Error(err)) } diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go new file mode 100644 index 000000000..0ad6c3287 --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go @@ -0,0 +1,19 @@ +package datastore + +import "time" + +const ( + TableNameWriteLock = "write_locks" +) + +// WriteLock WriteMarker lock +type WriteLock struct { + AllocationID string `gorm:"primaryKey, column:allocation_id"` + SessionID string `gorm:"column:session_id"` + CreatedAt time.Time `gorm:"column:created_at"` +} + +// TableName get table name of migrate +func (WriteLock) TableName() string { + return TableNameWriteLock +} diff --git a/code/go/0chain.net/blobbercore/handler/context.go b/code/go/0chain.net/blobbercore/handler/context.go new file mode 100644 index 000000000..2475b8527 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/context.go @@ -0,0 +1,171 @@ +package handler + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "net/http" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/errors" + "github.com/gorilla/mux" +) + +// Context api context +type Context struct { + context.Context + + // ClientID client wallet id + ClientID string + // ClientKey client wallet public key + ClientKey string + // AllocationTx optional. allcation id in request + AllocationTx string + // Signature optional. signature in request + Signature string + + Allocation *allocation.Allocation + + Store datastore.Store + Request *http.Request + + StatusCode int +} + +// FormValue get value from form data +func (c *Context) FormValue(key string) string { + return c.Request.FormValue(key) +} + +// FormTime get time from form data +func (c *Context) FormTime(key string) *time.Time { + value := c.Request.FormValue(key) + if len(value) == 0 { + return nil + } + + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil + } + + t := time.Unix(seconds, 0) + return &t +} + +type ErrorResponse struct { + Error string +} + +// WithHandler process handler to respond request +func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all. + if r.Method == "OPTIONS" { + w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") + w.Header().Set("Access-Control-Allow-Headers", "*") + return + } + + if r.Method == http.MethodPost || r.Method == http.MethodPut || r.Method == http.MethodPatch { + ct := r.Header.Get("Content-Type") + if ct == "application/x-www-form-urlencoded" { + r.ParseForm() //nolint: errcheck + } else { + r.ParseMultipartForm(FormFileParseMaxMemory) //nolint: errcheck + } + + } + + w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all. + w.Header().Set("Content-Type", "application/json") + + ctx, err := WithAuth(r) + statusCode := ctx.StatusCode + + if err != nil { + if statusCode == 0 { + statusCode = http.StatusInternalServerError + } + + buf, _ := json.Marshal(err) + http.Error(w, string(buf), statusCode) + return + } + + result, err := handler(ctx) + statusCode = ctx.StatusCode + + if err != nil { + if statusCode == 0 { + statusCode = http.StatusInternalServerError + } + + buf, _ := json.Marshal(err) + http.Error(w, string(buf), statusCode) + return + } + + if statusCode == 0 { + statusCode = http.StatusOK + } + w.WriteHeader(statusCode) + + if result != nil { + json.NewEncoder(w).Encode(result) //nolint + } + + } +} + +// WithAuth verify alloation and signature +func WithAuth(r *http.Request) (*Context, error) { + + ctx := &Context{ + Context: context.TODO(), + Request: r, + Store: datastore.GetStore(), + } + + var vars = mux.Vars(r) + + ctx.ClientID = r.Header.Get(common.ClientHeader) + ctx.ClientKey = r.Header.Get(common.ClientKeyHeader) + ctx.AllocationTx = vars["allocation"] + ctx.Signature = r.Header.Get(common.ClientSignatureHeader) + + if len(ctx.AllocationTx) > 0 { + alloc, err := allocation.GetOrCreate(ctx, ctx.Store, ctx.AllocationTx) + + if err != nil { + if errors.Is(common.ErrBadRequest, err) { + ctx.StatusCode = http.StatusBadRequest + + } else { + ctx.StatusCode = http.StatusInternalServerError + } + + return ctx, err + } + + ctx.Allocation = alloc + + valid, err := verifySignatureFromRequest(ctx.AllocationTx, ctx.Signature, alloc.OwnerPublicKey) + + if !valid { + ctx.StatusCode = http.StatusBadRequest + return ctx, errors.Throw(common.ErrBadRequest, "invalid signature "+ctx.Signature) + } + + if err != nil { + ctx.StatusCode = http.StatusInternalServerError + return ctx, errors.ThrowLog(err.Error(), common.ErrInternal, "invalid signature "+ctx.Signature) + } + } + + return ctx, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index f10dbb5a2..bc2eba128 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -75,6 +75,11 @@ func SetupHandlers(r *mux.Router) { //marketplace related r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.ToJSONResponse(WithConnection(MarketPlaceShareInfoHandler))) + + // lightweight http handler without heavy postgres transaction to improve performance + + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go new file mode 100644 index 000000000..02196fd5f --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go @@ -0,0 +1,35 @@ +//go:build !integration_tests +// +build !integration_tests + +package handler + +import ( + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" +) + +var WriteMarkerMutext = &writemarker.Mutex{} + +// LockWriteMarker try to lock writemarker for specified allocation id, and return latest RefTree +func LockWriteMarker(ctx *Context) (interface{}, error) { + sessionID := ctx.FormValue("session_id") + requestTime := ctx.FormTime("request_time") + + result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationTx, sessionID, requestTime) + if err != nil { + return nil, err + } + + return result, nil +} + +// UnlockWriteMarker release WriteMarkerMutex +func UnlockWriteMarker(ctx *Context) (interface{}, error) { + sessionID := ctx.FormValue("session_id") + + err := WriteMarkerMutext.Unlock(ctx, ctx.AllocationTx, sessionID) + if err != nil { + return nil, err + } + + return nil, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go new file mode 100644 index 000000000..48779982f --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go @@ -0,0 +1,92 @@ +package handler + +import ( + "bytes" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" + "github.com/gorilla/mux" + "github.com/stretchr/testify/require" +) + +func TestWriteMarkerHandlers_Lock(t *testing.T) { + + datastore.UseMocket(true) + + r := mux.NewRouter() + SetupHandlers(r) + + body := &bytes.Buffer{} + formWriter := multipart.NewWriter(body) + + now := time.Now() + + formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck + formWriter.Close() + + req, err := http.NewRequest(http.MethodPost, "/v1/writemarker/lock/{allocation}", body) + if err != nil { + t.Fatal(err) + } + + req.Header.Set("Content-Type", formWriter.FormDataContentType()) + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(WithHandler(func(ctx *Context) (interface{}, error) { + ctx.AllocationTx = "TestHandlers_Lock_allocation_id" + return LockWriteMarker(ctx) + })) + + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + + var result writemarker.LockResult + + err = json.Unmarshal(rr.Body.Bytes(), &result) + require.Nil(t, err) + + require.Equal(t, writemarker.LockStatusOK, result.Status) +} + +func TestWriteMarkerHandlers_Unlock(t *testing.T) { + datastore.UseMocket(true) + + r := mux.NewRouter() + SetupHandlers(r) + + body := &bytes.Buffer{} + formWriter := multipart.NewWriter(body) + + now := time.Now() + + formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck + formWriter.Close() + + req, err := http.NewRequest(http.MethodDelete, "/v1/writemarker/lock/{allocation}", body) + if err != nil { + t.Fatal(err) + } + + req.Header.Set("Content-Type", formWriter.FormDataContentType()) + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(WithHandler(func(ctx *Context) (interface{}, error) { + ctx.AllocationTx = "TestHandlers_Unlock_allocation_id" + return UnlockWriteMarker(ctx) + })) + + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + +} diff --git a/code/go/0chain.net/blobbercore/reference/dao.go b/code/go/0chain.net/blobbercore/reference/dao.go new file mode 100644 index 000000000..363ce4825 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/dao.go @@ -0,0 +1,73 @@ +package reference + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/errors" + "gorm.io/gorm" +) + +// LoadRootNode load root node with its descendant nodes +func LoadRootNode(ctx context.Context, allocationID string) (*HashNode, error) { + + db := datastore.GetStore().GetDB() + + db = db.Where("allocation_id = ? and deleted_at IS NULL", allocationID) + + db = db.Order("level desc, path") + + dict := make(map[string][]*HashNode) + + var nodes []*HashNode + // it is better to load them in batched if there are a lot of objects in db + err := db.FindInBatches(&nodes, 100, func(tx *gorm.DB, batch int) error { + // batch processing found records + for _, object := range nodes { + dict[object.ParentPath] = append(dict[object.ParentPath], object) + + for _, child := range dict[object.Path] { + object.AddChild(child) + } + } + + return nil + }).Error + + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + // create empty dir if root is missing + if len(dict) == 0 { + return &HashNode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil + } + + rootNodes, ok := dict[""] + + if ok { + if len(rootNodes) == 1 { + return rootNodes[0], nil + } + + return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") + } + + return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") +} + +const ( + SQLWhereGetByAllocationTxAndPath = "reference_objects.allocation_id = ? and reference_objects.path = ? and deleted_at is NULL" +) + +// DryRun Creates a prepared statement when executing any SQL and caches them to speed up future calls +// https://gorm.io/docs/performance.html#Caches-Prepared-Statement +func DryRun(db *gorm.DB) { + + // https://gorm.io/docs/session.html#DryRun + // Session mode + //tx := db.Session(&gorm.Session{PrepareStmt: true, DryRun: true}) + + // use Table instead of Model to reduce reflect times +} diff --git a/code/go/0chain.net/blobbercore/reference/entity.go b/code/go/0chain.net/blobbercore/reference/entity.go new file mode 100644 index 000000000..2fae95c28 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/entity.go @@ -0,0 +1,69 @@ +package reference + +import ( + "strconv" + "strings" + + "gorm.io/datatypes" +) + +// HashNode ref node in hash tree +type HashNode struct { + // hash data + AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` + Type string `gorm:"column:type" json:"type,omitempty"` + Name string `gorm:"column:name" json:"name,omitempty"` + Path string `gorm:"column:path" json:"path,omitempty"` + ContentHash string `gorm:"column:content_hash" json:"content_hash,omitempty"` + MerkleRoot string `gorm:"column:merkle_root" json:"merkle_root,omitempty"` + ActualFileHash string `gorm:"column:actual_file_hash" json:"actual_file_hash,omitempty"` + Attributes datatypes.JSON `gorm:"column:attributes" json:"attributes,omitempty"` + ChunkSize int64 `gorm:"column:chunk_size" json:"chunk_size,omitempty"` + Size int64 `gorm:"column:size" json:"size,omitempty"` + ActualFileSize int64 `gorm:"column:actual_file_size" json:"actual_file_size,omitempty"` + + // other data + ParentPath string `gorm:"parent_path" json:"-"` + Children []*HashNode `gorm:"-" json:"children,omitempty"` +} + +// TableName get table name of Ref +func (HashNode) TableName() string { + return TableNameReferenceObjects +} + +func (n *HashNode) AddChild(c *HashNode) { + if n.Children == nil { + n.Children = make([]*HashNode, 0, 10) + } + + n.Children = append(n.Children, c) +} + +// GetLookupHash get lookuphash +func (n *HashNode) GetLookupHash() string { + return GetReferenceLookup(n.AllocationID, n.Path) +} + +// GetHashCode get hash code +func (n *HashNode) GetHashCode() string { + + if len(n.Attributes) == 0 { + n.Attributes = datatypes.JSON("{}") + } + hashArray := []string{ + n.AllocationID, + n.Type, + n.Name, + n.Path, + strconv.FormatInt(n.Size, 10), + n.ContentHash, + n.MerkleRoot, + strconv.FormatInt(n.ActualFileSize, 10), + n.ActualFileHash, + string(n.Attributes), + strconv.FormatInt(n.ChunkSize, 10), + } + + return strings.Join(hashArray, ":") +} diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go new file mode 100644 index 000000000..120163930 --- /dev/null +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -0,0 +1,164 @@ +package writemarker + +import ( + "context" + "sync" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/errors" + "github.com/0chain/gosdk/constants" + "gorm.io/gorm" +) + +// LockStatus lock status +type LockStatus int + +const ( + LockStatusFailed LockStatus = iota + LockStatusPending + LockStatusOK +) + +type LockResult struct { + Status LockStatus `json:"status,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + RootNode *reference.HashNode `json:"root_node,omitempty"` +} + +// Mutex WriteMarker mutex +type Mutex struct { + sync.Mutex +} + +// Lock +func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, requestTime *time.Time) (*LockResult, error) { + m.Mutex.Lock() + defer m.Mutex.Unlock() + + if len(allocationID) == 0 { + return nil, errors.Throw(constants.ErrInvalidParameter, "allocationID") + } + + if len(sessionID) == 0 { + return nil, errors.Throw(constants.ErrInvalidParameter, "sessionID") + } + + if requestTime == nil { + return nil, errors.Throw(constants.ErrInvalidParameter, "requestTime") + } + + now := time.Now() + if requestTime.After(now.Add(config.Configuration.WriteMarkerLockTimeout)) { + return nil, errors.Throw(constants.ErrInvalidParameter, "requestTime") + } + + db := datastore.GetStore().GetDB() + + var lock datastore.WriteLock + err := db.Table(datastore.TableNameWriteLock).Where("allocation_id=?", allocationID).First(&lock).Error + if err != nil { + // new lock + if errors.Is(err, gorm.ErrRecordNotFound) { + lock = datastore.WriteLock{ + AllocationID: allocationID, + SessionID: sessionID, + CreatedAt: *requestTime, + } + + err = db.Create(&lock).Error + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + rootNode, err := reference.LoadRootNode(ctx, allocationID) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + return &LockResult{ + Status: LockStatusOK, + CreatedAt: lock.CreatedAt, + RootNode: rootNode, + }, nil + + } + + //native postgres error + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + + } + + timeout := lock.CreatedAt.Add(config.Configuration.WriteMarkerLockTimeout) + + // locked, but it is timeout + if now.After(timeout) { + + lock.SessionID = sessionID + lock.CreatedAt = *requestTime + + err = db.Save(&lock).Error + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + rootNode, err := reference.LoadRootNode(ctx, allocationID) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + return &LockResult{ + Status: LockStatusOK, + CreatedAt: lock.CreatedAt, + RootNode: rootNode, + }, nil + + } + + //try lock by same session, return old lock directly + if lock.SessionID == sessionID && lock.CreatedAt.Equal(*requestTime) { + rootNode, err := reference.LoadRootNode(ctx, allocationID) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + return &LockResult{ + Status: LockStatusOK, + CreatedAt: lock.CreatedAt, + RootNode: rootNode, + }, nil + } + + // pending + return &LockResult{ + Status: LockStatusPending, + CreatedAt: lock.CreatedAt, + }, nil + +} + +func (*Mutex) Unlock(ctx context.Context, allocationID string, sessionID string) error { + + if len(allocationID) == 0 { + return nil + } + + if len(sessionID) == 0 { + return nil + } + + db := datastore.GetStore().GetDB() + + err := db.Where("allocation_id = ? and session_id =? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + return errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go new file mode 100644 index 000000000..0c09c4f47 --- /dev/null +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -0,0 +1,199 @@ +package writemarker + +import ( + "context" + "testing" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestMutext_LockShouldWork(t *testing.T) { + + datastore.UseMocket(false) + + config.Configuration.WriteMarkerLockTimeout = 30 * time.Second + + m := &Mutex{} + now := time.Now() + + tests := []struct { + name string + allocationID string + sessionID string + requestTime time.Time + mock func() + assert func(*testing.T, *LockResult, error) + }{ + { + name: "Lock should work", + allocationID: "lock_allocation_id", + sessionID: "lock_session_id", + requestTime: now, + mock: func() { + + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, err) + require.Equal(test, LockStatusOK, r.Status) + }, + }, + { + name: "retry lock by same request should work if it is not timeout", + allocationID: "lock_same_allocation_id", + sessionID: "lock_same_session_id", + requestTime: now, + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). + WithArgs("lock_same_allocation_id"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "lock_same_allocation_id", + "session_id": "lock_same_session_id", + "created_at": now, + }, + }) + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, err) + require.Equal(test, LockStatusOK, r.Status) + }, + }, + { + name: "lock should be pending if it already is locked by other session ", + allocationID: "lock_allocation_id", + sessionID: "lock_pending_session_id", + requestTime: time.Now(), + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). + WithArgs("lock_allocation_id"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "lock_allocation_id", + "session_id": "lock_session_id", + "created_at": time.Now().Add(-5 * time.Second), + }, + }) + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, err) + require.Equal(test, LockStatusPending, r.Status) + }, + }, + { + name: "lock should ok if it is timeout", + allocationID: "lock_timeout_allocation_id", + sessionID: "lock_timeout_2nd_session_id", + requestTime: now, + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). + WithArgs("lock_timeout_allocation_id"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "lock_timeout_allocation_id", + "session_id": "lock_timeout_1st_session_id", + "created_at": time.Now().Add(31 * time.Second), + }, + }) + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, err) + require.Equal(test, LockStatusPending, r.Status) + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + + it.assert(test, r, err) + + }, + ) + + } + +} + +func TestMutext_LockShouldNotWork(t *testing.T) { + + datastore.UseMocket(true) + + config.Configuration.WriteMarkerLockTimeout = 30 * time.Second + + m := &Mutex{} + now := time.Now() + + tests := []struct { + name string + allocationID string + sessionID string + requestTime time.Time + mock func() + assert func(*testing.T, *LockResult, error) + }{ + { + name: "Lock should not work if request_time is timeout", + allocationID: "lock_allocation_id", + sessionID: "lock_session_id", + requestTime: time.Now().Add(31 * time.Second), + mock: func() { + config.Configuration.WriteMarkerLockTimeout = 30 * time.Second + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, r) + require.NotNil(test, err) + }, + }, + { + name: "retry lock by same request should not work if it is timeout", + allocationID: "lock_same_timeout_allocation_id", + sessionID: "lock_same_timeout_session_id", + requestTime: now, + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). + WithArgs("lock_same_timeout_allocation_id"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "lock_same_timeout_allocation_id", + "session_id": "lock_same_timeout_session_id", + "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), + }, + }) + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.NotNil(test, err) + require.Nil(test, r) + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + + it.assert(test, r, err) + + }, + ) + + } +} diff --git a/code/go/0chain.net/core/common/constants.go b/code/go/0chain.net/core/common/constants.go new file mode 100644 index 000000000..2345aebe1 --- /dev/null +++ b/code/go/0chain.net/core/common/constants.go @@ -0,0 +1,35 @@ +package common + +import "errors" + +var ( + // ErrBadDataStore bad db operation + ErrBadDataStore = errors.New("datastore: bad db operation") + + // ErrInvalidParameter parameter is not specified or invalid + ErrInvalidParameter = errors.New("invalid parameter") + + // ErrUnableHash failed to hash with unknown exception + ErrUnableHash = errors.New("unable to hash") + + // ErrUnableWriteFile failed to write bytes to file + ErrUnableWriteFile = errors.New("unable to write file") + + // ErrNotImplemented feature/method is not implemented yet + ErrNotImplemented = errors.New("not implemented") + + // ErrInvalidOperation failed to invoke a method + ErrInvalidOperation = errors.New("invalid operation") + + // ErrBadRequest bad request + ErrBadRequest = errors.New("bad request") + + // ErrUnknown unknown exception + ErrUnknown = errors.New("unknown") + + // ErrInternal an unknown internal server error + ErrInternal = errors.New("internal") + + // ErrEntityNotFound entity can't found in db + ErrEntityNotFound = errors.New("entity not found") +) diff --git a/go.mod b/go.mod index 58784a021..d9e844bef 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.5.1-0.20220211020847-2b54b2fa404f + github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/didip/tollbooth/v6 v6.1.1 + github.com/didip/tollbooth/v6 v6.1.2 github.com/go-ini/ini v1.55.0 // indirect github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 @@ -14,7 +14,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.15.0 - github.com/jackc/pgx/v4 v4.14.1 // indirect + github.com/jackc/pgx/v4 v4.15.0 // indirect github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/mapstructure v1.4.3 @@ -26,9 +26,10 @@ require ( github.com/stretchr/testify v1.7.0 go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.21.0 - golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068 google.golang.org/grpc v1.44.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 @@ -36,8 +37,8 @@ require ( gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d - gorm.io/driver/postgres v1.2.3 - gorm.io/gorm v1.22.5 + gorm.io/driver/postgres v1.3.1 + gorm.io/gorm v1.23.1 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index ace573d4c..a154c6548 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.5.1-0.20220211020847-2b54b2fa404f h1:dLjkhdiMIY9Rs3mUlEAPqAQKf+lE1NLKFd+bzVkQyzk= -github.com/0chain/gosdk v1.5.1-0.20220211020847-2b54b2fa404f/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= +github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 h1:cOqg66kR646dc1NNgduE0Ridlg0bDDvhxOPC9rhrNWk= +github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -226,8 +226,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M= github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY= -github.com/didip/tollbooth/v6 v6.1.1 h1:Nt7PvWLa9Y94OrykXsFNBinVRQIu8xdy4avpl99Dc1M= -github.com/didip/tollbooth/v6 v6.1.1/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s= +github.com/didip/tollbooth/v6 v6.1.2 h1:Kdqxmqw9YTv0uKajBUiWQg+GURL/k4vy9gmLCL01PjQ= +github.com/didip/tollbooth/v6 v6.1.2/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -555,8 +555,9 @@ github.com/jackc/pgconn v1.6.1/go.mod h1:g8mKMqmSUO6AzAvha7vy07g1rbGOlc7iF0nU0ei github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8= github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -588,9 +589,9 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkAL github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= github.com/jackc/pgtype v1.4.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.9.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgtype v1.9.1 h1:MJc2s0MFS8C3ok1wQTdQxWuXQcB6+HwAm5x1CzW7mf0= github.com/jackc/pgtype v1.9.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -599,15 +600,16 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.7.1/go.mod h1:nu42q3aPjuC1M0Nak4bnoprKlXPINqopEKqbq5AZSC4= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.14.0/go.mod h1:jT3ibf/A0ZVCp89rtCIN0zCJxcE74ypROmHEZYsG/j8= -github.com/jackc/pgx/v4 v4.14.1 h1:71oo1KAGI6mXhLiTMn6iDFcp3e7+zon/capWjl2OEFU= github.com/jackc/pgx/v4 v4.14.1/go.mod h1:RgDuE4Z34o7XE92RpLsvFiOEfrAUT0Xt2KxvX73W06M= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= @@ -616,7 +618,6 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -1082,8 +1083,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1331,8 +1332,9 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1627,8 +1629,8 @@ gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d/go.mod h1:n2DTgk9at7cr/CWOT gorm.io/driver/mysql v0.3.1 h1:yvUT7Q0I3B9EHJ67NSp6cHbVwcdDHhVUsDAUiFFxRk0= gorm.io/driver/mysql v0.3.1/go.mod h1:A7H1JD9dKdcjeUTpTuWKEC+E1a74qzW7/zaXqKaTbfM= gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs= -gorm.io/driver/postgres v1.2.3 h1:f4t0TmNMy9gh3TU2PX+EppoA6YsgFnyq8Ojtddb42To= -gorm.io/driver/postgres v1.2.3/go.mod h1:pJV6RgYQPG47aM1f0QeOzFH9HxQc8JcmAgjRCgS0wjs= +gorm.io/driver/postgres v1.3.1 h1:Pyv+gg1Gq1IgsLYytj/S2k7ebII3CzEdpqQkPOdH24g= +gorm.io/driver/postgres v1.3.1/go.mod h1:WwvWOuR9unCLpGWCL6Y3JOeBWvbKi6JLhayiVclSZZU= gorm.io/driver/sqlite v1.0.8 h1:omllgSb7/eh9D6lGvLZOdU1ZElxdXuO3dn3Rk+dQxUE= gorm.io/driver/sqlite v1.0.8/go.mod h1:xkm8/CEmA3yc4zRd0pdCqm43BjO8Hm6avfTpxWb/7c4= gorm.io/driver/sqlserver v0.2.5 h1:o/MXpn9/BB68RXEEQzfhsSL382yEqUtdCiGIuCspmkY= @@ -1636,9 +1638,8 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.5 h1:lYREBgc02Be/5lSCTuysZZDb6ffL2qrat6fg9CFbvXU= -gorm.io/gorm v1.22.5/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.1 h1:aj5IlhDzEPsoIyOPtTRVI+SyaN1u6k613sbt4pwbxG0= +gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 1449981a644c82d2a9c95cfd7747adcfd29869d7 Mon Sep 17 00:00:00 2001 From: prince Date: Wed, 23 Feb 2022 04:38:38 +0530 Subject: [PATCH 108/161] delete file query update. (#542) --- .../allocation/deletefilechange.go | 2 +- .../allocation/file_changer_update.go | 27 ++ .../allocation/updatefilechange_test.go | 315 ++++++++++++++++++ .../blobbercore/filestore/fs_store.go | 7 +- .../blobbercore/filestore/mock_store.go | 24 +- .../object_operation_handler_bench_test.go | 2 +- .../blobbercore/reference/object.go | 5 +- 7 files changed, 375 insertions(+), 7 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index d878ffd21..618791755 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -58,7 +58,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) for contenthash := range nf.ContentHash { var count int64 - err := db.Table((&reference.Ref{}).TableName()).Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash}).Count(&count).Error + err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error if err == nil && count == 0 { Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 53e7f7fcd..1af88e223 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -5,6 +5,8 @@ import ( "encoding/json" "path/filepath" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" @@ -15,6 +17,7 @@ import ( ) type UpdateFileChanger struct { + deleteHash map[string]bool BaseFileChanger } @@ -59,6 +62,15 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati return nil, common.NewError("file_not_found", "File to update not found in blobber") } existingRef := dirRef.Children[idx] + // remove changed thumbnail and files + nf.deleteHash = make(map[string]bool) + if existingRef.ThumbnailHash != "" && existingRef.ThumbnailHash != nf.ThumbnailHash { + nf.deleteHash[existingRef.ThumbnailHash] = true + } + if existingRef.ContentHash != "" && existingRef.ContentHash != nf.Hash { + nf.deleteHash[existingRef.ContentHash] = true + } + existingRef.ActualFileHash = nf.ActualHash existingRef.ActualFileSize = nf.ActualSize existingRef.MimeType = nf.MimeType @@ -84,6 +96,21 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati return rootRef, err } +func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + for contenthash := range nf.deleteHash { + var count int64 + err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error + if err == nil && count == 0 { + Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) + if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { + Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err)) + } + } + } + return nf.BaseFileChanger.CommitToFileStore(ctx) +} + func (nf *UpdateFileChanger) Marshal() (string, error) { ret, err := json.Marshal(nf) if err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go new file mode 100644 index 000000000..cf7164bf9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -0,0 +1,315 @@ +package allocation + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk/zboxcore/client" + mocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/grpc/metadata" +) + +func init() { + logging.Logger = zap.NewNop() +} + +func TestBlobberCore_UpdateFile(t *testing.T) { + + sch := zcncrypto.NewSignatureScheme("bls0chain") + mnemonic := "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" + _, err := sch.RecoverKeys(mnemonic) + if err != nil { + t.Fatal(err) + } + ts := time.Now().Add(time.Hour) + alloc := makeTestAllocation(common.Timestamp(ts.Unix())) + alloc.OwnerPublicKey = sch.GetPublicKey() + alloc.OwnerID = client.GetClientID() + + testCases := []struct { + name string + context metadata.MD + allocChange *AllocationChange + path string + filename string + allocRoot string + thumbnailHash string + hash string + allocationID string + expectedMessage string + expectingError bool + setupDbMock func() + initDir, expectedDir map[string]map[string]bool + }{ + { + name: "Update thumbnail hash", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "thumbnail_hash", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash_old", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash_old": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + }, + { + name: "Update content hash", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "thumbnail_hash", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash_old", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash_old": true, + "thumbnail_hash": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + }, + { + name: "Remove thumbnail", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + }, + }, + }, + } + + for _, tc := range testCases { + datastore.MocketTheStore(t, true) + filestore.UseMock(tc.initDir) + tc.setupDbMock() + + ctx := context.TODO() + db := datastore.GetStore().GetDB().Begin() + ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + + change := &UpdateFileChanger{ + BaseFileChanger: BaseFileChanger{ + Path: tc.path, + Filename: tc.filename, + ActualSize: 2310, + ActualThumbnailSize: 92, + ActualThumbnailHash: tc.thumbnailHash, + Attributes: reference.Attributes{WhoPaysForReads: common.WhoPaysOwner}, + AllocationID: tc.allocationID, + Hash: tc.hash, + Size: 2310, + ThumbnailHash: tc.thumbnailHash, + ThumbnailSize: 92, + ChunkSize: 65536, + IsFinal: true, + }, + } + + _, err := func() (*reference.Ref, error) { + resp, err := change.ProcessChange(ctx, tc.allocChange, tc.allocRoot) + if err != nil { + return nil, err + } + + err = change.CommitToFileStore(ctx) + return resp, err + }() + + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + if tc.expectingError && strings.Contains(tc.expectedMessage, err.Error()) { + t.Fatal("expected error " + tc.expectedMessage) + break + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + require.EqualValues(t, tc.expectedDir, tc.initDir) + } +} diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index fd1e7c9f9..18cb3e8dc 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -603,8 +603,11 @@ func (fs *FileFSStore) DownloadFromCloud(fileHash, filePath string) error { } func (fs *FileFSStore) RemoveFromCloud(fileHash string) error { - if _, err := fs.Minio.StatObject(MinioConfig.BucketName, fileHash, minio.StatObjectOptions{}); err == nil { - return fs.Minio.RemoveObject(MinioConfig.BucketName, fileHash) + if fs != nil && fs.Minio != nil { + _, err := fs.Minio.StatObject(MinioConfig.BucketName, fileHash, minio.StatObjectOptions{}) + if err == nil { + return fs.Minio.RemoveObject(MinioConfig.BucketName, fileHash) + } } return nil } diff --git a/code/go/0chain.net/blobbercore/filestore/mock_store.go b/code/go/0chain.net/blobbercore/filestore/mock_store.go index 3eb356f76..4ff6c312e 100644 --- a/code/go/0chain.net/blobbercore/filestore/mock_store.go +++ b/code/go/0chain.net/blobbercore/filestore/mock_store.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "io" "mime/multipart" @@ -12,15 +13,17 @@ import ( ) type MockStore struct { + d map[string]map[string]bool } var mockStore *MockStore -func UseMock() { +func UseMock(initData map[string]map[string]bool) { if mockStore == nil { - mockStore = &MockStore{} + mockStore = &MockStore{d: initData} } + mockStore.d = initData fileStore = mockStore } @@ -67,6 +70,7 @@ func (ms *MockStore) GetFileBlock(allocationID string, fileData *FileInputData, } func (ms *MockStore) CommitWrite(allocationID string, fileData *FileInputData, connectionID string) (bool, error) { + ms.addFileInDataObj(allocationID, fileData.Hash) return true, nil } @@ -74,8 +78,13 @@ func (ms *MockStore) GetFileBlockForChallenge(allocationID string, fileData *Fil return nil, nil, constants.ErrNotImplemented } func (ms *MockStore) DeleteFile(allocationID, contentHash string) error { + if ms.d == nil || ms.d[allocationID] == nil || !ms.d[allocationID][contentHash] { + return errors.New("file not available related to content") + } + delete(ms.d[allocationID], contentHash) return nil } + func (ms *MockStore) GetTotalDiskSizeUsed() (int64, error) { return 0, constants.ErrNotImplemented } @@ -97,3 +106,14 @@ func (ms *MockStore) DownloadFromCloud(fileHash, filePath string) error { func (ms *MockStore) SetupAllocation(allocationID string, skipCreate bool) (*StoreAllocation, error) { return nil, constants.ErrNotImplemented } + +func (ms *MockStore) addFileInDataObj(allocationID, contentHash string) { + if contentHash == "" { + return + } + if ms.d == nil { + ms.d = make(map[string]map[string]bool, 0) + } + dataObj := ms.d[allocationID] + dataObj[contentHash] = true +} diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go index 4763c426d..526addeeb 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go @@ -107,7 +107,7 @@ func BenchmarkUploadFileWithNoDisk(b *testing.B) { //GB := 1024 * MB datastore.UseMocket(false) - filestore.UseMock() + filestore.UseMock(nil) blobber := mock.NewBlobberClient() allocationID := "benchmark_uploadfile" diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 28575d38d..b0f265f4a 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -71,7 +71,7 @@ func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[str GetTransaction(ctx) var deletedObjects []*Ref - txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "type"}}}) + txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "thumbnail_hash"}, {Name: "type"}}}) path = filepath.Join("/", path) txDelete = txDelete.Where("allocation_id = ? and deleted_at IS NULL and (path LIKE ? or path = ?) and path != ? ", allocationID, (path + "%"), path, "/") @@ -85,6 +85,9 @@ func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[str for _, it := range deletedObjects { if it.Type == FILE { deletedFiles[it.ContentHash] = true + if it.ThumbnailHash != "" { + deletedFiles[it.ThumbnailHash] = true + } } } From 36745908024c6865221d37ed82c4861cf216fa58 Mon Sep 17 00:00:00 2001 From: Kishan-Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Date: Fri, 25 Feb 2022 09:50:02 +0530 Subject: [PATCH 109/161] Update PR template (#553) --- PULL_REQUEST_TEMPLATE.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 7e5cc0d4c..a79a1ff7e 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,15 @@ +A brief description of the changes in this PR: + + Tasks to complete before merging PR: - [ ] Ensure system tests are passing. If not [Run them manually](https://github.com/0chain/blobber/actions/workflows/system_tests.yml) to check for any regressions :clipboard: - [ ] Do any new system tests need added to test this change? do any existing system tests need updated? If so create a PR at [0chain/system_test](https://github.com/0chain/system_test) -- [ ] Merge your system tests PR to master BEFORE merging this PR +- [ ] Merge your system tests PR to master AFTER merging this PR Associated PRs (Link as appropriate): -- zbox: -- zwalelet: -- gosdk: - 0chain: -- system_test: \ No newline at end of file +- gosdk: +- system_test: +- zboxcli: +- zwalletcli: +- Other: ... From 88ba0c5ded1f7cd43ee1115fbf0d2b0c8c042209 Mon Sep 17 00:00:00 2001 From: prince Date: Sun, 27 Feb 2022 22:27:59 +0530 Subject: [PATCH 110/161] change data reading from form to header for download. (#550) --- .../0chain.net/blobbercore/convert/convert.go | 57 +- .../0chain.net/blobbercore/handler/context.go | 10 +- .../handler/download_request_header.go | 113 ++ .../handler/handler_download_test.go | 1076 +++++++++++++++++ .../blobbercore/handler/handler_test.go | 921 +------------- .../0chain.net/blobbercore/handler/helper.go | 13 + .../handler/object_operation_handler.go | 62 +- .../handler/object_operation_handler_test.go | 29 +- .../blobbercore/handler/storage_handler.go | 5 + 9 files changed, 1244 insertions(+), 1042 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/download_request_header.go create mode 100644 code/go/0chain.net/blobbercore/handler/handler_download_test.go diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index ab399015f..0db96ed93 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -6,7 +6,6 @@ import ( "encoding/json" "mime/multipart" "net/http" - "strings" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" @@ -483,57 +482,19 @@ func WriteFileGRPCToHTTP(req *blobbergrpc.UploadFileRequest) (*http.Request, err } func DownloadFileGRPCToHTTP(req *blobbergrpc.DownloadFileRequest) (*http.Request, error) { - body := bytes.NewBuffer([]byte{}) - writer := multipart.NewWriter(body) - err := writer.WriteField("path", req.Path) + r, err := http.NewRequest("GET", "", nil) if err != nil { return nil, err } - err = writer.WriteField("path_hash", req.PathHash) - if err != nil { - return nil, err - } - - err = writer.WriteField("rx_pay", req.RxPay) - if err != nil { - return nil, err - } - - err = writer.WriteField("block_num", req.BlockNum) - if err != nil { - return nil, err - } - - err = writer.WriteField("num_blocks", req.NumBlocks) - if err != nil { - return nil, err - } - - err = writer.WriteField("read_marker", req.ReadMarker) - if err != nil { - return nil, err - } - - err = writer.WriteField("auth_token", req.AuthToken) - if err != nil { - return nil, err - } - - err = writer.WriteField("content", req.Content) - if err != nil { - return nil, err - } - - writer.Close() - - r, err := http.NewRequest("POST", "", strings.NewReader(body.String())) - if err != nil { - return nil, err - } - - r.Header.Set("Content-Type", writer.FormDataContentType()) - + r.Header.Set("X-Path", req.Path) + r.Header.Set("X-Path-Hash", req.PathHash) + r.Header.Set("X-Rxpay", req.RxPay) + r.Header.Set("X-Block-Num", req.BlockNum) + r.Header.Set("X-Num-Blocks", req.NumBlocks) + r.Header.Set("X-Read-Marker", req.ReadMarker) + r.Header.Set("X-Auth-Token", req.AuthToken) + r.Header.Set("X-Mode", req.Content) return r, nil } diff --git a/code/go/0chain.net/blobbercore/handler/context.go b/code/go/0chain.net/blobbercore/handler/context.go index 2475b8527..0f27146a5 100644 --- a/code/go/0chain.net/blobbercore/handler/context.go +++ b/code/go/0chain.net/blobbercore/handler/context.go @@ -71,15 +71,7 @@ func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.Re return } - if r.Method == http.MethodPost || r.Method == http.MethodPut || r.Method == http.MethodPatch { - ct := r.Header.Get("Content-Type") - if ct == "application/x-www-form-urlencoded" { - r.ParseForm() //nolint: errcheck - } else { - r.ParseMultipartForm(FormFileParseMaxMemory) //nolint: errcheck - } - - } + TryParseForm(r) w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all. w.Header().Set("Content-Type", "application/json") diff --git a/code/go/0chain.net/blobbercore/handler/download_request_header.go b/code/go/0chain.net/blobbercore/handler/download_request_header.go new file mode 100644 index 000000000..1503fd28a --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/download_request_header.go @@ -0,0 +1,113 @@ +package handler + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/errors" +) + +// DownloadRequestHeader metedata of download request +type DownloadRequestHeader struct { + req *http.Request + allocationID string + PathHash string + Path string + BlockNum int64 + NumBlocks int64 + ReadMarker readmarker.ReadMarker + AuthToken string + RxPay bool + DownloadMode string +} + +func FromDownloadRequest(allocationID string, req *http.Request) (*DownloadRequestHeader, error) { + if allocationID == "" { + return nil, errors.Throw(common.ErrInvalidParameter, "allocationID") + } + + if req == nil { + return nil, errors.Throw(common.ErrInvalidParameter, "req") + } + + dr := &DownloadRequestHeader{ + allocationID: allocationID, + req: req, + } + + err := dr.Parse() + if err != nil { + return nil, err + } + + return dr, nil +} + +func (dr *DownloadRequestHeader) Parse() error { + if dr.req == nil { + return errors.Throw(common.ErrInvalidParameter, "req") + } + + pathHash := dr.Get("X-Path-Hash") + path := dr.Get("X-Path") + if pathHash == "" { + if path == "" { + return errors.Throw(common.ErrInvalidParameter, "X-Path") + } + pathHash = reference.GetReferenceLookup(dr.allocationID, path) + } + + dr.PathHash = pathHash + dr.Path = path + + blockNum := dr.GetInt64("X-Block-Num", -1) + if blockNum <= 0 { + return errors.Throw(common.ErrInvalidParameter, "X-Block-Num") + } + dr.BlockNum = blockNum + + numBlocks := dr.GetInt64("X-Num-Blocks", 1) + if numBlocks <= 0 { + return errors.Throw(common.ErrInvalidParameter, "X-Num-Blocks") + } + dr.NumBlocks = numBlocks + + readMarker := dr.Get("X-Read-Marker") + + if readMarker == "" { + return errors.Throw(common.ErrInvalidParameter, "X-Read-Marker") + } + + err := json.Unmarshal([]byte(readMarker), &dr.ReadMarker) + if err != nil { + return errors.Throw(common.ErrInvalidParameter, "X-Read-Marker") + } + + dr.AuthToken = dr.Get("X-Auth-Token") + + dr.RxPay = dr.Get("X-Rxpay") == "true" + dr.DownloadMode = dr.Get("X-Mode") + + return nil +} + +func (dr *DownloadRequestHeader) Get(key string) string { + if dr.req == nil { + return "" + } + return dr.req.Header.Get(key) +} + +func (dr *DownloadRequestHeader) GetInt64(key string, defaultValue int64) int64 { + v := dr.Get(key) + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return defaultValue + } + + return i +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_download_test.go b/code/go/0chain.net/blobbercore/handler/handler_download_test.go new file mode 100644 index 000000000..1fe163206 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_download_test.go @@ -0,0 +1,1076 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "regexp" + "testing" + "time" + + "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk/zboxcore/client" + zencryption "github.com/0chain/gosdk/zboxcore/encryption" + "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk/zboxcore/marker" + "github.com/DATA-DOG/go-sqlmock" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" +) + +func setupDownloadHandlers() (*mux.Router, map[string]string) { + router := mux.NewRouter() + + dPath := "/v1/file/download/{allocation}" + dName := "Download" + router.HandleFunc(dPath, common.UserRateLimit( + common.ToJSONResponse( + WithConnection(DownloadHandler), + ), + ), + ).Name(dName) + + return router, map[string]string{ + dPath: dName, + } +} + +func getEncryptionScheme(mnemonic string) (zencryption.EncryptionScheme, error) { + encscheme := zencryption.NewEncryptionScheme() + if _, err := encscheme.Initialize(mnemonic); err != nil { + return nil, err + } + encscheme.InitForEncryption("filetype:audio") + return encscheme, nil +} + +func TestHandlers_Download_Requiring_Signature(t *testing.T) { + setup(t) + + clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}` + guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}` + + require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) + clients := client.GetClients() + + ownerClient, guestClient := clients[0], clients[1] + + ownerScheme, err := getEncryptionScheme(ownerClient.Mnemonic) + if err != nil { + t.Fatal(err) + } + + guestScheme, err := getEncryptionScheme(guestClient.Mnemonic) + if err != nil { + t.Fatal(err) + } + // require.NoError(t, client.PopulateClient(clientJson, "bls0chain")) + // setupEncryptionScheme() + + router, handlers := setupDownloadHandlers() + + sch := zcncrypto.NewSignatureScheme("bls0chain") + //sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" + _, err = sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") + if err != nil { + t.Fatal(err) + } + + ts := time.Now().Add(time.Hour) + alloc := makeTestAllocation(common.Timestamp(ts.Unix())) + alloc.OwnerPublicKey = ownerClient.Keys[0].PublicKey + alloc.OwnerID = ownerClient.ClientID + + type ( + args struct { + w *httptest.ResponseRecorder + r *http.Request + } + test struct { + name string + args args + alloc *allocation.Allocation + setupDbMock func(mock sqlmock.Sqlmock) + begin func() + end func() + wantCode int + wantBody string + } + ) + negativeTests := make([]test, 0) + for _, name := range handlers { + if !isEndpointRequireSignature(name) { + continue + } + + baseSetupDbMock := func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows([]string{"id", "tx", "expiration_date", "owner_public_key", "owner_id"}). + AddRow(alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + mock.ExpectCommit() + } + + emptySignature := test{ + name: name + "_Empty_Signature", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + url, err := router.Get(name).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + method := http.MethodGet + if !isEndpointAllowGetReq(name) { + method = http.MethodPost + } + r, err := http.NewRequest(method, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + return r + }(), + }, + alloc: alloc, + setupDbMock: baseSetupDbMock, + wantCode: http.StatusBadRequest, + wantBody: "{\"code\":\"invalid_signature\",\"error\":\"invalid_signature: Invalid signature\"}\n\n", + } + negativeTests = append(negativeTests, emptySignature) + + wrongSignature := test{ + name: name + "_Wrong_Signature", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + url, err := router.Get(name).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + method := http.MethodGet + if !isEndpointAllowGetReq(name) { + method = http.MethodPost + } + r, err := http.NewRequest(method, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash("another data") + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + + return r + }(), + }, + alloc: alloc, + setupDbMock: baseSetupDbMock, + wantCode: http.StatusBadRequest, + wantBody: "{\"code\":\"invalid_signature\",\"error\":\"invalid_signature: Invalid signature\"}\n\n", + } + negativeTests = append(negativeTests, wrongSignature) + } + + positiveTests := []test{ + { + name: "DownloadFile_Record_Not_Found", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + remotePath := "/file.txt" + + rm := &marker.ReadMarker{} + rm.ClientID = ownerClient.ClientID + rm.ClientPublicKey = ownerClient.ClientKey + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(ownerClient, rm.GetHash()) + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + require.NoError(t, err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("X-Path-Hash", fileref.GetReferenceLookup(alloc.Tx, remotePath)) + r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash).WillReturnError(gorm.ErrRecordNotFound) + + }, + wantCode: http.StatusBadRequest, + wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: invalid file path: record not found\"}\n\n", + }, + { + name: "DownloadFile_Unencrypted_return_file", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + remotePath := "/file.txt" + + rm := &marker.ReadMarker{} + rm.ClientID = ownerClient.ClientID + rm.ClientPublicKey = ownerClient.ClientKey + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(ownerClient, rm.GetHash()) + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("X-Path-Hash", fileref.GetReferenceLookup(alloc.Tx, remotePath)) + r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "lookup_hash", "content_hash"}). + AddRow("/file.txt", "f", filePathHash, "abcd"), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). + WithArgs(ownerClient.ClientID). + WillReturnError(gorm.ErrRecordNotFound) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(ownerClient.ClientID). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(ownerClient.ClientID), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`(UPDATE "read_markers" SET)(.+)`). + WithArgs(ownerClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "\"bW9jaw==\"\n", //base64encoded for mock string + }, + { + name: "DownloadFile_Encrypted_Permission_Denied_Unshared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + remotePath := "/file.txt" + + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, ownerClient.Keys[0].PublicKey) + if err != nil { + t.Fatal(err) + } + + rm := &marker.ReadMarker{} + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(guestClient, rm.GetHash()) + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set("X-Path-Hash", pathHash) + r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", "qCj3sXXeXUAByi1ERIbcfXzWN75dyocYzyRXnkStXio=", 65536), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnError(gorm.ErrRecordNotFound) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(client.GetClientID()), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(client.GetClientID(), filePathHash). + WillReturnError(gorm.ErrRecordNotFound) + + mock.ExpectCommit() + }, + wantCode: http.StatusBadRequest, + wantBody: "{\"error\":\"client does not have permission to download the file. share does not exist\"}\n\n", + }, + { + name: "DownloadFile_Encrypted_Permission_Allowed_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + remotePath := "/file.txt" + + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") + if err != nil { + t.Fatal(err) + } + rm := &marker.ReadMarker{} + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(guestClient, rm.GetHash()) + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + r, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set("X-Path-Hash", pathHash) + r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + + header := make([]byte, EncryptionHeaderSize) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + fmt.Println("Encrypted data: ", string(data)) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), 65536), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). + WithArgs(guestClient.ClientID). + WillReturnError(gorm.ErrRecordNotFound) + + guestPublicEncryptedKey, err := guestScheme.GetPublicKey() + if err != nil { + t.Fatal(err) + } + reEncryptionKey, err := ownerScheme.GetReGenKey(guestPublicEncryptedKey, "filetype:audio") + + if err != nil { + t.Fatal(err) + } + fmt.Printf("\n\nencryptedKey: %v\tgpbk: %v\treKey: %v\n\n", ownerScheme.GetEncryptedKey(), guestPublicEncryptedKey, reEncryptionKey) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(guestClient.ClientID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, guestPublicEncryptedKey), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(guestClient.ClientID). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(guestClient.ClientID), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "", + }, + { + name: "DownloadFile_Encrypted_InSharedFolder_Permission_Allowed_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + remotePath := "/" + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") + if err != nil { + t.Fatal(err) + } + rm := &marker.ReadMarker{} + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = ownerClient.ClientID + rm.Signature, err = signHash(guestClient, rm.GetHash()) + if err != nil { + t.Fatal(err) + } + + rmData, err := json.Marshal(rm) + require.NoError(t, err) + + r, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("X-Path-Hash", filePathHash) + r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + + header := make([]byte, EncryptionHeaderSize) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/", fileref.CHUNK_SIZE), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). + WithArgs(guestClient.ClientID). + WillReturnError(gorm.ErrRecordNotFound) + + rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/", "d", rootPathHash, rootPathHash, "content_hash", "", "."), + ) + + gpbk, err := guestScheme.GetPublicKey() + if err != nil { + t.Fatal(err) + } + + reEncryptionKey, _ := ownerScheme.GetReGenKey(gpbk, "filetype:audio") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(guestClient.ClientID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, gpbk), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(guestClient.ClientID). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(guestClient.ClientID), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "", + }, + { + name: "DownloadFile_Encrypted_InSharedFolderSubdirectory_Permission_Allowed_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + remotePath := "/folder1" + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1/subfolder1/file.txt") + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") + if err != nil { + t.Fatal(err) + } + rm := &marker.ReadMarker{} + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = alloc.OwnerID + rm.Signature, err = signHash(guestClient, rm.GetHash()) + if err != nil { + t.Fatal(err) + } + + rmData, err := json.Marshal(rm) + require.NoError(t, err) + + r, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("X-Path-Hash", filePathHash) + r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + + header := make([]byte, EncryptionHeaderSize) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1/subfolder1/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). + AddRow("/folder1/subfolder1/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/folder1/subfolder1", filestore.CHUNK_SIZE), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). + WithArgs(guestClient.ClientID). + WillReturnError(gorm.ErrRecordNotFound) + + rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "."), + ) + + gpbk, err := guestScheme.GetPublicKey() + if err != nil { + t.Fatal(err) + } + + reEncryptionKey, _ := ownerScheme.GetReGenKey(gpbk, "filetype:audio") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(guestClient.ClientID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, gpbk), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(guestClient.ClientID). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(guestClient.ClientID), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "", + }, + { + name: "DownloadFile_Encrypted_InSharedFolder_WrongFilePath_Permission_Rejected_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + remotePath := "/folder1" + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder2/subfolder1/file.txt") + authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") + if err != nil { + t.Fatal(err) + } + rm := &marker.ReadMarker{} + rm.ClientID = guestClient.ClientID + rm.ClientPublicKey = guestClient.ClientKey + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = alloc.OwnerID + rm.Signature, err = signHash(guestClient, rm.GetHash()) + if err != nil { + t.Fatal(err) + } + + rmData, err := json.Marshal(rm) + require.NoError(t, err) + + r, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set("X-Path-Hash", filePathHash) + r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, guestClient.ClientID) + r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + + header := make([]byte, EncryptionHeaderSize) + copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder2/subfolder1/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/folder2/subfolder1", fileref.CHUNK_SIZE), + ) + + rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, rootPathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). + AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "/"), + ) + + }, + wantCode: http.StatusBadRequest, + wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: cannot verify auth ticket: invalid_parameters: Auth ticket is not valid for the resource being requested\"}\n\n", + }, + } + + tests := append(positiveTests, negativeTests...) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mock := datastore.MockTheStore(t) + test.setupDbMock(mock) + + if test.begin != nil { + test.begin() + } + router.ServeHTTP(test.args.w, test.args.r) + if test.end != nil { + test.end() + } + + fmt.Printf("\nResponse body: %v", test.args.w.Body.String()) + assert.Equal(t, test.wantCode, test.args.w.Result().StatusCode) + if test.wantCode != http.StatusOK || test.wantBody != "" { + assert.Equal(t, test.wantBody, test.args.w.Body.String()) + } + }) + } + + curDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(curDir + "/tmp"); err != nil { + t.Fatal(err) + } +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 49615e78e..f09bd17d8 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -17,7 +17,6 @@ import ( "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/gosdk/zboxcore/client" - zencryption "github.com/0chain/gosdk/zboxcore/encryption" "github.com/0chain/gosdk/zboxcore/fileref" "github.com/0chain/gosdk/zboxcore/marker" "github.com/0chain/gosdk/zcncore" @@ -88,15 +87,6 @@ func signHash(client *client.Client, hash string) (string, error) { return retSignature, nil } -func getEncryptionScheme(mnemonic string) (zencryption.EncryptionScheme, error) { - encscheme := zencryption.NewEncryptionScheme() - if _, err := encscheme.Initialize(mnemonic); err != nil { - return nil, err - } - encscheme.InitForEncryption("filetype:audio") - return encscheme, nil -} - func init() { resetMockFileBlock() common.ConfigRateLimits() @@ -328,25 +318,13 @@ func TestHandlers_Requiring_Signature(t *testing.T) { require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) clients := client.GetClients() - ownerClient, guestClient := clients[0], clients[1] - - ownerScheme, err := getEncryptionScheme(ownerClient.Mnemonic) - if err != nil { - t.Fatal(err) - } - - guestScheme, err := getEncryptionScheme(guestClient.Mnemonic) - if err != nil { - t.Fatal(err) - } - // require.NoError(t, client.PopulateClient(clientJson, "bls0chain")) - // setupEncryptionScheme() + ownerClient := clients[0] router, handlers := setupHandlers() sch := zcncrypto.NewSignatureScheme("bls0chain") //sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" - _, err = sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") + _, err := sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") if err != nil { t.Fatal(err) } @@ -1495,904 +1473,9 @@ func TestHandlers_Requiring_Signature(t *testing.T) { wantCode: http.StatusOK, wantBody: "{\"message\":\"Path not found\",\"status\":404}\n", }, - { - name: "DownloadFile_Record_Not_Found", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - body := bytes.NewBuffer(nil) - formWriter := multipart.NewWriter(body) - remotePath := "/file.txt" - - require.NoError(t, formWriter.WriteField("path_hash", fileref.GetReferenceLookup(alloc.Tx, remotePath))) - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - rm := &marker.ReadMarker{} - rm.ClientID = ownerClient.ClientID - rm.ClientPublicKey = ownerClient.ClientKey - rm.BlobberID = "" - rm.AllocationID = alloc.ID - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(ownerClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) - if err := formWriter.Close(); err != nil { - t.Fatal(err) - } - r, err := http.NewRequest(http.MethodPost, url.String(), body) - r.Header.Add("Content-Type", formWriter.FormDataContentType()) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("Content-Type", formWriter.FormDataContentType()) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) - - return r - }(), - }, - alloc: alloc, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash).WillReturnError(gorm.ErrRecordNotFound) - - }, - wantCode: http.StatusBadRequest, - wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: invalid file path: record not found\"}\n\n", - }, - { - name: "DownloadFile_Unencrypted_return_file", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - body := bytes.NewBuffer(nil) - formWriter := multipart.NewWriter(body) - remotePath := "/file.txt" - - require.NoError(t, formWriter.WriteField("path_hash", fileref.GetReferenceLookup(alloc.Tx, remotePath))) - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - rm := &marker.ReadMarker{} - rm.ClientID = ownerClient.ClientID - rm.ClientPublicKey = ownerClient.ClientKey - rm.BlobberID = "" - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(ownerClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) - if err := formWriter.Close(); err != nil { - t.Fatal(err) - } - r, err := http.NewRequest(http.MethodPost, url.String(), body) - r.Header.Add("Content-Type", formWriter.FormDataContentType()) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("Content-Type", formWriter.FormDataContentType()) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) - - return r - }(), - }, - alloc: alloc, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "lookup_hash", "content_hash"}). - AddRow("/file.txt", "f", filePathHash, "abcd"), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(ownerClient.ClientID). - WillReturnError(gorm.ErrRecordNotFound) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(ownerClient.ClientID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(ownerClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`(UPDATE "read_markers" SET)(.+)`). - WithArgs(ownerClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - - mock.ExpectCommit() - }, - wantCode: http.StatusOK, - wantBody: "\"bW9jaw==\"\n", //base64encoded for mock string - }, - { - name: "DownloadFile_Encrypted_Permission_Denied_Unshared_File", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - body := bytes.NewBuffer(nil) - formWriter := multipart.NewWriter(body) - remotePath := "/file.txt" - - pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) - require.NoError(t, formWriter.WriteField("path_hash", pathHash)) - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, ownerClient.Keys[0].PublicKey) - if err != nil { - t.Fatal(err) - } - - require.NoError(t, formWriter.WriteField("auth_token", authTicket)) - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = "" - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) - if err := formWriter.Close(); err != nil { - t.Fatal(err) - } - r, err := http.NewRequest(http.MethodPost, url.String(), body) - r.Header.Add("Content-Type", formWriter.FormDataContentType()) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("Content-Type", formWriter.FormDataContentType()) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, guestClient.ClientID) - r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) - - return r - }(), - }, - alloc: alloc, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", "qCj3sXXeXUAByi1ERIbcfXzWN75dyocYzyRXnkStXio=", 65536), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(client.GetClientID()). - WillReturnError(gorm.ErrRecordNotFound) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(client.GetClientID()). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(client.GetClientID()), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs(client.GetClientID(), filePathHash). - WillReturnError(gorm.ErrRecordNotFound) - - mock.ExpectCommit() - }, - wantCode: http.StatusBadRequest, - wantBody: "{\"error\":\"client does not have permission to download the file. share does not exist\"}\n\n", - }, - { - name: "DownloadFile_Encrypted_Permission_Allowed_shared_File", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - body := bytes.NewBuffer(nil) - formWriter := multipart.NewWriter(body) - remotePath := "/file.txt" - - pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) - require.NoError(t, formWriter.WriteField("path_hash", pathHash)) - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") - if err != nil { - t.Fatal(err) - } - require.NoError(t, formWriter.WriteField("auth_token", authTicket)) - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = "" - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) - if err := formWriter.Close(); err != nil { - t.Fatal(err) - } - r, err := http.NewRequest(http.MethodPost, url.String(), body) - r.Header.Add("Content-Type", formWriter.FormDataContentType()) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("Content-Type", formWriter.FormDataContentType()) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, guestClient.ClientID) - r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) - - return r - }(), - }, - alloc: alloc, - begin: func() { - dataToEncrypt := "data_to_encrypt" - encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) - if err != nil { - t.Fatal(err) - } - - header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) - data := append(header, encMsg.EncryptedData...) - fmt.Println("Encrypted data: ", string(data)) - setMockFileBlock(data) - }, - end: func() { - resetMockFileBlock() - }, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), 65536), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(guestClient.ClientID). - WillReturnError(gorm.ErrRecordNotFound) - - guestPublicEncryptedKey, err := guestScheme.GetPublicKey() - if err != nil { - t.Fatal(err) - } - reEncryptionKey, err := ownerScheme.GetReGenKey(guestPublicEncryptedKey, "filetype:audio") - - if err != nil { - t.Fatal(err) - } - fmt.Printf("\n\nencryptedKey: %v\tgpbk: %v\treKey: %v\n\n", ownerScheme.GetEncryptedKey(), guestPublicEncryptedKey, reEncryptionKey) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs(guestClient.ClientID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). - AddRow(reEncryptionKey, guestPublicEncryptedKey), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(guestClient.ClientID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(guestClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - - mock.ExpectCommit() - }, - wantCode: http.StatusOK, - wantBody: "", - }, - { - name: "DownloadFile_Encrypted_InSharedFolder_Permission_Allowed_shared_File", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - body := bytes.NewBuffer(nil) - formWriter := multipart.NewWriter(body) - - remotePath := "/" - pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") - require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) - - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") - if err != nil { - t.Fatal(err) - } - require.NoError(t, formWriter.WriteField("auth_token", authTicket)) - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = "" - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - - rmData, err := json.Marshal(rm) - require.NoError(t, err) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) - if err := formWriter.Close(); err != nil { - t.Fatal(err) - } - - r, err := http.NewRequest(http.MethodPost, url.String(), body) - r.Header.Add("Content-Type", formWriter.FormDataContentType()) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("Content-Type", formWriter.FormDataContentType()) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, guestClient.ClientID) - r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) - - return r - }(), - }, - alloc: alloc, - begin: func() { - dataToEncrypt := "data_to_encrypt" - encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) - if err != nil { - t.Fatal(err) - } - - header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) - data := append(header, encMsg.EncryptedData...) - setMockFileBlock(data) - }, - end: func() { - resetMockFileBlock() - }, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/", fileref.CHUNK_SIZE), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(guestClient.ClientID). - WillReturnError(gorm.ErrRecordNotFound) - - rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, rootPathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). - AddRow("/", "d", rootPathHash, rootPathHash, "content_hash", "", "."), - ) - - gpbk, err := guestScheme.GetPublicKey() - if err != nil { - t.Fatal(err) - } - - reEncryptionKey, _ := ownerScheme.GetReGenKey(gpbk, "filetype:audio") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs(guestClient.ClientID, rootPathHash). - WillReturnRows( - sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). - AddRow(reEncryptionKey, gpbk), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(guestClient.ClientID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(guestClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - - mock.ExpectCommit() - }, - wantCode: http.StatusOK, - wantBody: "", - }, - { - name: "DownloadFile_Encrypted_InSharedFolderSubdirectory_Permission_Allowed_shared_File", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - body := bytes.NewBuffer(nil) - formWriter := multipart.NewWriter(body) - - remotePath := "/folder1" - pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1/subfolder1/file.txt") - require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) - - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") - if err != nil { - t.Fatal(err) - } - require.NoError(t, formWriter.WriteField("auth_token", authTicket)) - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = "" - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.OwnerID = alloc.OwnerID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - - rmData, err := json.Marshal(rm) - require.NoError(t, err) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) - if err := formWriter.Close(); err != nil { - t.Fatal(err) - } - - r, err := http.NewRequest(http.MethodPost, url.String(), body) - r.Header.Add("Content-Type", formWriter.FormDataContentType()) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("Content-Type", formWriter.FormDataContentType()) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, guestClient.ClientID) - r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) - - return r - }(), - }, - alloc: alloc, - begin: func() { - dataToEncrypt := "data_to_encrypt" - encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) - if err != nil { - t.Fatal(err) - } - - header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) - data := append(header, encMsg.EncryptedData...) - setMockFileBlock(data) - }, - end: func() { - resetMockFileBlock() - }, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1/subfolder1/file.txt") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). - AddRow("/folder1/subfolder1/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/folder1/subfolder1", filestore.CHUNK_SIZE), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "collaborators" WHERE`)). - WithArgs(guestClient.ClientID). - WillReturnError(gorm.ErrRecordNotFound) - - rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, rootPathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). - AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "."), - ) - - gpbk, err := guestScheme.GetPublicKey() - if err != nil { - t.Fatal(err) - } - - reEncryptionKey, _ := ownerScheme.GetReGenKey(gpbk, "filetype:audio") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). - WithArgs(guestClient.ClientID, rootPathHash). - WillReturnRows( - sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). - AddRow(reEncryptionKey, gpbk), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(guestClient.ClientID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(guestClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(guestClient.ClientKey, alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - - mock.ExpectCommit() - }, - wantCode: http.StatusOK, - wantBody: "", - }, - { - name: "DownloadFile_Encrypted_InSharedFolder_WrongFilePath_Permission_Rejected_shared_File", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - body := bytes.NewBuffer(nil) - formWriter := multipart.NewWriter(body) - - remotePath := "/folder1" - pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder2/subfolder1/file.txt") - require.NoError(t, formWriter.WriteField("path_hash", filePathHash)) - - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) - authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") - if err != nil { - t.Fatal(err) - } - require.NoError(t, formWriter.WriteField("auth_token", authTicket)) - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = "" - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.OwnerID = alloc.OwnerID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - - rmData, err := json.Marshal(rm) - require.NoError(t, err) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) - if err := formWriter.Close(); err != nil { - t.Fatal(err) - } - - r, err := http.NewRequest(http.MethodPost, url.String(), body) - r.Header.Add("Content-Type", formWriter.FormDataContentType()) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("Content-Type", formWriter.FormDataContentType()) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, guestClient.ClientID) - r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) - - return r - }(), - }, - alloc: alloc, - begin: func() { - dataToEncrypt := "data_to_encrypt" - encMsg, err := ownerScheme.Encrypt([]byte(dataToEncrypt)) - if err != nil { - t.Fatal(err) - } - - header := make([]byte, EncryptionHeaderSize) - copy(header, encMsg.MessageChecksum+encMsg.OverallChecksum) - data := append(header, encMsg.EncryptedData...) - setMockFileBlock(data) - }, - end: func() { - resetMockFileBlock() - }, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder2/subfolder1/file.txt") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path", "chunk_size"}). - AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", ownerScheme.GetEncryptedKey(), "/folder2/subfolder1", fileref.CHUNK_SIZE), - ) - - rootPathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, rootPathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key", "parent_path"}). - AddRow("/folder1", "d", rootPathHash, rootPathHash, "content_hash", "", "/"), - ) - - }, - wantCode: http.StatusBadRequest, - wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: cannot verify auth ticket: invalid_parameters: Auth ticket is not valid for the resource being requested\"}\n\n", - }, } tests := append(positiveTests, negativeTests...) - // tests := positiveTests for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/code/go/0chain.net/blobbercore/handler/helper.go b/code/go/0chain.net/blobbercore/handler/helper.go index 408aed256..651077ccb 100644 --- a/code/go/0chain.net/blobbercore/handler/helper.go +++ b/code/go/0chain.net/blobbercore/handler/helper.go @@ -1,6 +1,7 @@ package handler import ( + "net/http" "time" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -15,3 +16,15 @@ func checkValidDate(s, dateLayOut string) error { } return nil } + +// TryParseForm try populates r.Form and r.PostForm. +func TryParseForm(r *http.Request) { + if r.Method == http.MethodPost || r.Method == http.MethodPut || r.Method == http.MethodPatch { + ct := r.Header.Get("Content-Type") + if ct == "application/x-www-form-urlencoded" { + r.ParseForm() //nolint: errcheck + } else { + r.ParseMultipartForm(FormFileParseMaxMemory) //nolint: errcheck + } + } +} diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 30d17f85d..4fd456ff1 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -192,54 +192,20 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r return nil, common.NewErrorf("download_file", "invalid allocation id passed: %v", err) } - if err = r.ParseMultipartForm(FormFileParseMaxMemory); err != nil { - Logger.Info("download_file - request_parse_error", zap.Error(err)) - return nil, common.NewErrorf("download_file", "request_parse_error: %v", err) - } - - pathHash, err := pathHashFromReq(r, alloc.ID) + dr, err := FromDownloadRequest(allocationTx, r) if err != nil { - return nil, common.NewError("download_file", "invalid path") - } - - var blockNumStr = r.FormValue("block_num") - if blockNumStr == "" { - return nil, common.NewError("download_file", "no block number") - } - - var blockNum int64 - blockNum, err = strconv.ParseInt(blockNumStr, 10, 64) - if err != nil || blockNum < 1 { - return nil, common.NewError("download_file", "invalid block number") - } - - numBlocks := int64(1) - numBlocksStr := r.FormValue("num_blocks") - - if numBlocksStr != "" { - numBlocks, err = strconv.ParseInt(numBlocksStr, 10, 64) - if err != nil || numBlocks <= 0 { - return nil, common.NewError("download_file", "invalid number of blocks") - } - } - - readMarkerString := r.FormValue("read_marker") - readMarker := new(readmarker.ReadMarker) - - if err := json.Unmarshal([]byte(readMarkerString), readMarker); err != nil { - return nil, common.NewErrorf("download_file", "invalid parameters, "+ - "error parsing the readmarker for download: %v", err) + return nil, err } rmObj := new(readmarker.ReadMarkerEntity) - rmObj.LatestRM = readMarker + rmObj.LatestRM = &dr.ReadMarker if err = rmObj.VerifyMarker(ctx, alloc); err != nil { return nil, common.NewErrorf("download_file", "invalid read marker, "+"failed to verify the read marker: %v", err) } // get file reference - fileref, err := reference.GetReferenceFromLookupHash(ctx, alloc.ID, pathHash) + fileref, err := reference.GetReferenceFromLookupHash(ctx, alloc.ID, dr.PathHash) if err != nil { return nil, common.NewErrorf("download_file", "invalid file path: %v", err) } @@ -264,7 +230,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r var shareInfo *reference.ShareInfo if !(isOwner || isCollaborator) { - authTokenString := r.FormValue("auth_token") + authTokenString := dr.AuthToken if authTokenString == "" { return nil, common.NewError("invalid_client", "authticket is required") } @@ -283,11 +249,11 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r } // set payer: check for command line payer flag (--rx_pay) - if r.FormValue("rx_pay") == "true" { + if dr.RxPay { payerID = clientID } - readMarker.AuthTicket = datatypes.JSON(authTokenString) + dr.ReadMarker.AuthTicket = datatypes.JSON(authTokenString) // check for file payer flag if fileAttrs, err := fileref.GetAttributes(); err != nil { @@ -315,7 +281,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r } } - if latestRM != nil && latestRM.ReadCounter+(numBlocks) != readMarker.ReadCounter { + if latestRM != nil && latestRM.ReadCounter+(dr.NumBlocks) != dr.ReadMarker.ReadCounter { return &blobberhttp.DownloadResponse{ Success: false, LatestRM: latestRM, @@ -325,14 +291,14 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r } // check out read pool tokens if read_price > 0 - err = readPreRedeem(ctx, alloc, numBlocks, pendNumBlocks, payerID) + err = readPreRedeem(ctx, alloc, dr.NumBlocks, pendNumBlocks, payerID) if err != nil { return nil, common.NewErrorf("download_file", "pre-redeeming read marker: %v", err) } // reading is allowed var ( - downloadMode = r.FormValue("content") + downloadMode = dr.DownloadMode respData []byte ) if downloadMode == DownloadContentThumb { @@ -342,7 +308,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r fileData.Hash = fileref.ThumbnailHash fileData.OnCloud = fileref.OnCloud fileData.ChunkSize = fileref.ChunkSize - respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, blockNum, numBlocks) + respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, dr.BlockNum, dr.NumBlocks) if err != nil { return nil, common.NewErrorf("download_file", "couldn't get thumbnail block: %v", err) } @@ -354,14 +320,14 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (r fileData.OnCloud = fileref.OnCloud fileData.ChunkSize = fileref.ChunkSize - respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, blockNum, numBlocks) + respData, err = filestore.GetFileStore().GetFileBlock(alloc.ID, fileData, dr.BlockNum, dr.NumBlocks) if err != nil { return nil, common.NewErrorf("download_file", "couldn't get file block: %v", err) } } - readMarker.PayerID = payerID - err = readmarker.SaveLatestReadMarker(ctx, readMarker, latestRM == nil) + dr.ReadMarker.PayerID = payerID + err = readmarker.SaveLatestReadMarker(ctx, &dr.ReadMarker, latestRM == nil) if err != nil { Logger.Error(err.Error()) return nil, common.NewErrorf("download_file", "couldn't save latest read marker") diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index 71dc038c7..99b09f632 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -1,12 +1,10 @@ package handler import ( - "bytes" "context" "database/sql/driver" "encoding/json" "fmt" - "mime/multipart" "net/http" "net/http/httptest" "time" @@ -111,7 +109,7 @@ func TestDownloadFile(t *testing.T) { // reuse code from GOSDK, https://github.com/0chain/gosdk/blob/staging/zboxcore/sdk/blockdownloadworker.go#L150 var addToForm = func( t *testing.T, - formWriter *multipart.Writer, + req *http.Request, p parameters, ) *marker.ReadMarker { rm := &marker.ReadMarker{} @@ -126,14 +124,14 @@ func TestDownloadFile(t *testing.T) { require.NoError(t, err) rmData, err := json.Marshal(rm) require.NoError(t, err) - require.NoError(t, formWriter.WriteField("path_hash", p.inData.pathHash)) - require.NoError(t, formWriter.WriteField("path", p.inData.remotefilepath)) + req.Header.Set("X-Path-Hash", p.inData.pathHash) + req.Header.Set("X-Path", p.inData.remotefilepath) if p.inData.rxPay { - require.NoError(t, formWriter.WriteField("rx_pay", "true")) + req.Header.Set("X-Rxpay", "true") } - require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", p.inData.blockNum))) - require.NoError(t, formWriter.WriteField("num_blocks", fmt.Sprintf("%d", p.inData.numBlocks))) - require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + req.Header.Set("X-Block-Num", fmt.Sprintf("%d", p.inData.blockNum)) + req.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", p.inData.numBlocks)) + req.Header.Set("X-Read-Marker", string(rmData)) if p.useAuthTicket { authTicket := &marker.AuthTicket{ AllocationID: p.inData.allocationID, @@ -147,12 +145,11 @@ func TestDownloadFile(t *testing.T) { require.NoError(t, authTicket.Sign()) require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain")) authTicketBytes, _ := json.Marshal(authTicket) - require.NoError(t, formWriter.WriteField("auth_token", string(authTicketBytes))) + req.Header.Set("X-Auth-Token", string(authTicketBytes)) } if len(p.inData.contentMode) > 0 { - require.NoError(t, formWriter.WriteField("content", p.inData.contentMode)) + req.Header.Set("X-Mode", p.inData.contentMode) } - require.NoError(t, formWriter.Close()) return rm } @@ -432,12 +429,8 @@ func TestDownloadFile(t *testing.T) { } setupRequest := func(p parameters) (*http.Request, *marker.ReadMarker) { - body := new(bytes.Buffer) - formWriter := multipart.NewWriter(body) - rm := addToForm(t, formWriter, p) - req := httptest.NewRequest(http.MethodGet, "/v1/file/download/", body) - require.NoError(t, req.ParseForm()) - req.Header.Add("Content-Type", formWriter.FormDataContentType()) + req := httptest.NewRequest(http.MethodGet, "/v1/file/download/", nil) + rm := addToForm(t, req, p) return req, rm } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index e543f02cc..8c717cae2 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -984,6 +984,11 @@ func getPathHash(r *http.Request, allocationID string) (pathHash, path string, e pathHash = r.FormValue("path_hash") path = r.FormValue("path") + if pathHash == "" && path == "" { + pathHash = r.Header.Get("path_hash") + path = r.Header.Get("path") + } + if pathHash == "" { if path == "" { return "", "", common.NewError("invalid_parameters", "Invalid path") From cbfef7729af291d317f5f55faef90745d401b2bd Mon Sep 17 00:00:00 2001 From: prince Date: Mon, 28 Feb 2022 06:10:09 +0530 Subject: [PATCH 111/161] change statucode for delete file when file not found. (#551) --- .../blobbercore/handler/file_command_delete.go | 2 +- code/go/0chain.net/core/common/errors.go | 10 ++++++++-- code/go/0chain.net/core/common/handler.go | 8 ++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 1cdb288dd..786d5f7c0 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -31,7 +31,7 @@ func (cmd *FileCommandDelete) IsAuthorized(ctx context.Context, req *http.Reques cmd.exisitingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, path) if cmd.exisitingFileRef == nil { - return common.NewError("invalid_file", "File does not exist at path") + return common.NewErrorfWithStatusCode(204, "invalid_file", "File does not exist at path") } return nil diff --git a/code/go/0chain.net/core/common/errors.go b/code/go/0chain.net/core/common/errors.go index 9f09e5b06..052696808 100644 --- a/code/go/0chain.net/core/common/errors.go +++ b/code/go/0chain.net/core/common/errors.go @@ -6,8 +6,9 @@ import ( /*Error type for a new application error */ type Error struct { - Code string `json:"code,omitempty"` - Msg string `json:"msg"` + Code string `json:"code,omitempty"` + Msg string `json:"msg"` + StatusCode int `json:"status_code,omitempty"` } func (err *Error) Error() string { @@ -24,6 +25,11 @@ func NewErrorf(code, format string, args ...interface{}) *Error { return &Error{Code: code, Msg: fmt.Sprintf(format, args...)} } +/*NewErrorf - create a new error with format */ +func NewErrorfWithStatusCode(statusCode int, errCode, format string, args ...interface{}) *Error { + return &Error{StatusCode: statusCode, Code: errCode, Msg: fmt.Sprintf(format, args...)} +} + /*InvalidRequest - create error messages that are needed when validating request input */ func InvalidRequest(msg string) error { return NewError("invalid_request", fmt.Sprintf("Invalid request (%v)", msg)) diff --git a/code/go/0chain.net/core/common/handler.go b/code/go/0chain.net/core/common/handler.go index 14233ddac..8f24c11b0 100644 --- a/code/go/0chain.net/core/common/handler.go +++ b/code/go/0chain.net/core/common/handler.go @@ -62,14 +62,18 @@ func ToByteStream(handler JSONResponderF) ReqRespHandlerf { ctx := r.Context() data, err := handler(ctx, r) if err != nil { + statusCode := 400 if cerr, ok := err.(*Error); ok { w.Header().Set(AppErrorHeader, cerr.Code) + if cerr.StatusCode != 0 { + statusCode = cerr.StatusCode + } } if data != nil { responseString, _ := json.Marshal(data) - http.Error(w, string(responseString), 400) + http.Error(w, string(responseString), statusCode) } else { - http.Error(w, err.Error(), 400) + http.Error(w, err.Error(), statusCode) } } else if data != nil { rawdata, ok := data.([]byte) From b468fa92814c786e166682e6284d28ca05e0f4d2 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 13:51:54 +0800 Subject: [PATCH 112/161] Update PULL_REQUEST_TEMPLATE.md (#560) --- PULL_REQUEST_TEMPLATE.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index a79a1ff7e..9b5de71d5 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -1,12 +1,17 @@ -A brief description of the changes in this PR: +### Changes +### Fixes + + + +### Tests Tasks to complete before merging PR: - [ ] Ensure system tests are passing. If not [Run them manually](https://github.com/0chain/blobber/actions/workflows/system_tests.yml) to check for any regressions :clipboard: - [ ] Do any new system tests need added to test this change? do any existing system tests need updated? If so create a PR at [0chain/system_test](https://github.com/0chain/system_test) - [ ] Merge your system tests PR to master AFTER merging this PR -Associated PRs (Link as appropriate): +### Associated PRs (Link as appropriate): - 0chain: - gosdk: - system_test: From 73f9b736da71bf8596c4cfca53dff2f708c3a8cb Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 2 Mar 2022 16:13:43 +0800 Subject: [PATCH 113/161] fix/updated WriteMarkerMutex (#555) * feat(writemarker): updated WriteMarker mutex * feat(writemarker): updated CreateAt with int64 --- .../blobbercore/datastore/postgres_schema.go | 2 +- .../blobbercore/writemarker/mutex.go | 44 +++++-------------- .../blobbercore/writemarker/mutext_test.go | 28 ++++++------ 3 files changed, 27 insertions(+), 47 deletions(-) diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go index 0ad6c3287..d0c1bc2d4 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go @@ -9,7 +9,7 @@ const ( // WriteLock WriteMarker lock type WriteLock struct { AllocationID string `gorm:"primaryKey, column:allocation_id"` - SessionID string `gorm:"column:session_id"` + ConnectionID string `gorm:"column:connection_id"` CreatedAt time.Time `gorm:"column:created_at"` } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 120163930..daeaddab0 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -7,7 +7,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/errors" "github.com/0chain/gosdk/constants" @@ -24,9 +23,8 @@ const ( ) type LockResult struct { - Status LockStatus `json:"status,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` - RootNode *reference.HashNode `json:"root_node,omitempty"` + Status LockStatus `json:"status,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` } // Mutex WriteMarker mutex @@ -35,7 +33,7 @@ type Mutex struct { } // Lock -func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, requestTime *time.Time) (*LockResult, error) { +func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, requestTime *time.Time) (*LockResult, error) { m.Mutex.Lock() defer m.Mutex.Unlock() @@ -43,8 +41,8 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.Throw(constants.ErrInvalidParameter, "allocationID") } - if len(sessionID) == 0 { - return nil, errors.Throw(constants.ErrInvalidParameter, "sessionID") + if len(connectionID) == 0 { + return nil, errors.Throw(constants.ErrInvalidParameter, "connectionID") } if requestTime == nil { @@ -65,7 +63,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques if errors.Is(err, gorm.ErrRecordNotFound) { lock = datastore.WriteLock{ AllocationID: allocationID, - SessionID: sessionID, + ConnectionID: connectionID, CreatedAt: *requestTime, } @@ -74,15 +72,9 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, - RootNode: rootNode, + CreatedAt: lock.CreatedAt.Unix(), }, nil } @@ -97,7 +89,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques // locked, but it is timeout if now.After(timeout) { - lock.SessionID = sessionID + lock.ConnectionID = connectionID lock.CreatedAt = *requestTime err = db.Save(&lock).Error @@ -105,37 +97,25 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, - RootNode: rootNode, + CreatedAt: lock.CreatedAt.Unix(), }, nil } //try lock by same session, return old lock directly - if lock.SessionID == sessionID && lock.CreatedAt.Equal(*requestTime) { - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - + if lock.ConnectionID == connectionID && lock.CreatedAt.Equal(*requestTime) { return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, - RootNode: rootNode, + CreatedAt: lock.CreatedAt.Unix(), }, nil } // pending return &LockResult{ Status: LockStatusPending, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index 0c09c4f47..e36f0fe15 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -23,7 +23,7 @@ func TestMutext_LockShouldWork(t *testing.T) { tests := []struct { name string allocationID string - sessionID string + connectionID string requestTime time.Time mock func() assert func(*testing.T, *LockResult, error) @@ -31,7 +31,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "Lock should work", allocationID: "lock_allocation_id", - sessionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: now, mock: func() { @@ -44,7 +44,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "retry lock by same request should work if it is not timeout", allocationID: "lock_same_allocation_id", - sessionID: "lock_same_session_id", + connectionID: "lock_same_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -53,7 +53,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_allocation_id", - "session_id": "lock_same_session_id", + "connection_id": "lock_same_connection_id", "created_at": now, }, }) @@ -66,7 +66,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should be pending if it already is locked by other session ", allocationID: "lock_allocation_id", - sessionID: "lock_pending_session_id", + connectionID: "lock_pending_connection_id", requestTime: time.Now(), mock: func() { gomocket.Catcher.NewMock(). @@ -75,7 +75,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_allocation_id", - "session_id": "lock_session_id", + "connection_id": "lock_connection_id", "created_at": time.Now().Add(-5 * time.Second), }, }) @@ -88,7 +88,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should ok if it is timeout", allocationID: "lock_timeout_allocation_id", - sessionID: "lock_timeout_2nd_session_id", + connectionID: "lock_timeout_2nd_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -97,7 +97,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_timeout_allocation_id", - "session_id": "lock_timeout_1st_session_id", + "connection_id": "lock_timeout_1st_connection_id", "created_at": time.Now().Add(31 * time.Second), }, }) @@ -116,7 +116,7 @@ func TestMutext_LockShouldWork(t *testing.T) { if it.mock != nil { it.mock() } - r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + r, err := m.Lock(context.TODO(), it.allocationID, it.connectionID, &it.requestTime) it.assert(test, r, err) @@ -139,7 +139,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { tests := []struct { name string allocationID string - sessionID string + connectionID string requestTime time.Time mock func() assert func(*testing.T, *LockResult, error) @@ -147,7 +147,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "Lock should not work if request_time is timeout", allocationID: "lock_allocation_id", - sessionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: time.Now().Add(31 * time.Second), mock: func() { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second @@ -160,7 +160,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "retry lock by same request should not work if it is timeout", allocationID: "lock_same_timeout_allocation_id", - sessionID: "lock_same_timeout_session_id", + connectionID: "lock_same_timeout_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -169,7 +169,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_timeout_allocation_id", - "session_id": "lock_same_timeout_session_id", + "connection_id": "lock_same_timeout_connection_id", "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), }, }) @@ -188,7 +188,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { if it.mock != nil { it.mock() } - r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + r, err := m.Lock(context.TODO(), it.allocationID, it.connectionID, &it.requestTime) it.assert(test, r, err) From 8ff5a5410caa266196ad6948f661b1ec625fb26d Mon Sep 17 00:00:00 2001 From: prince Date: Wed, 2 Mar 2022 19:35:31 +0530 Subject: [PATCH 114/161] Feature/99 statuscode 204 for file not found (#559) * simplify logic for delete Signed-off-by: Pradip Parmar * commit handler logic update. Signed-off-by: Pradip Parmar * status handling for respond method Signed-off-by: Pradip Parmar * feat(delete): improved status code 204 logic Co-authored-by: Pradip Parmar Co-authored-by: Lz --- .../allocation/deletefilechange.go | 9 ++- .../handler/file_command_delete.go | 13 +++- .../handler/grpc_commit_handler.go | 2 +- .../0chain.net/blobbercore/handler/handler.go | 41 ++++++++++-- .../handler/object_operation_handler.go | 7 ++- code/go/0chain.net/core/common/constants.go | 3 + code/go/0chain.net/core/common/errors.go | 10 +-- code/go/0chain.net/core/common/handler.go | 62 +++++++++++++++++-- 8 files changed, 123 insertions(+), 24 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index 618791755..c5896c367 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -3,10 +3,12 @@ package allocation import ( "context" "encoding/json" + "os" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" @@ -56,15 +58,20 @@ func (nf *DeleteFileChange) DeleteTempFile() error { func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) + var errFileWasDeleted error for contenthash := range nf.ContentHash { var count int64 err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error if err == nil && count == 0 { Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { + if os.IsNotExist(err) { + errFileWasDeleted = common.ErrFileWasDeleted + continue + } Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err)) } } } - return nil + return errFileWasDeleted } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 786d5f7c0..60c26fdea 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -2,6 +2,7 @@ package handler import ( "context" + "errors" "net/http" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" @@ -9,6 +10,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/gosdk/constants" + "gorm.io/gorm" ) // FileCommandDelete command for deleting file @@ -28,12 +30,17 @@ func (cmd *FileCommandDelete) IsAuthorized(ctx context.Context, req *http.Reques if path == "" { return common.NewError("invalid_parameters", "Invalid path") } - cmd.exisitingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, path) - if cmd.exisitingFileRef == nil { - return common.NewErrorfWithStatusCode(204, "invalid_file", "File does not exist at path") + fileRef, err := reference.GetReference(ctx, allocationObj.ID, path) + if err != nil { + if errors.Is(gorm.ErrRecordNotFound, err) { + return common.ErrFileWasDeleted + } + return common.NewError("bad_db_operation", err.Error()) } + cmd.exisitingFileRef = fileRef + return nil } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go b/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go index cb522cbd2..0286f1abf 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go @@ -31,7 +31,7 @@ func (b *blobberGRPCService) Commit(ctx context.Context, req *blobbergrpc.Commit httpRequestWithMetaData(r, getGRPCMetaDataFromCtx(ctx), req.Allocation) r.Header.Set("Content-Type", writer.FormDataContentType()) - resp, err := CommitHandler(ctx, r) + resp, _, err := CommitHandler(ctx, r) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index bc2eba128..cb7a2710d 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -51,7 +51,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods(http.MethodPost, http.MethodOptions) r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods(http.MethodDelete, http.MethodOptions) - r.HandleFunc("/v1/connection/commit/{allocation}", common.ToJSONResponse(WithConnection(CommitHandler))) + r.HandleFunc("/v1/connection/commit/{allocation}", common.ToStatusCode(WithStatusConnection(CommitHandler))) r.HandleFunc("/v1/file/commitmetatxn/{allocation}", common.ToJSONResponse(WithConnection(CommitMetaTxnHandler))) r.HandleFunc("/v1/file/collaborator/{allocation}", common.ToJSONResponse(WithConnection(CollaboratorHandler))) r.HandleFunc("/v1/file/calculatehash/{allocation}", common.ToJSONResponse(WithConnection(CalculateHashHandler))) @@ -121,6 +121,34 @@ func WithConnection(handler common.JSONResponderF) common.JSONResponderF { } } +func WithStatusConnection(handler common.StatusCodeResponderF) common.StatusCodeResponderF { + return func(ctx context.Context, r *http.Request) (resp interface{}, statusCode int, err error) { + ctx = GetMetaDataStore().CreateTransaction(ctx) + resp, statusCode, err = handler(ctx, r) + + defer func() { + if err != nil { + var rollErr = GetMetaDataStore().GetTransaction(ctx). + Rollback().Error + if rollErr != nil { + Logger.Error("couldn't rollback", zap.Error(err)) + } + } + }() + + if err != nil { + Logger.Error("Error in handling the request." + err.Error()) + return + } + err = GetMetaDataStore().GetTransaction(ctx).Commit().Error + if err != nil { + return resp, statusCode, common.NewErrorf("commit_error", + "error committing to meta store: %v", err) + } + return + } +} + func setupHandlerContext(ctx context.Context, r *http.Request) context.Context { var vars = mux.Vars(r) ctx = context.WithValue(ctx, constants.ContextKeyClient, @@ -239,15 +267,20 @@ func ListHandler(ctx context.Context, r *http.Request) (interface{}, error) { } /*CommitHandler is the handler to respond to upload requests fro clients*/ -func CommitHandler(ctx context.Context, r *http.Request) (interface{}, error) { +func CommitHandler(ctx context.Context, r *http.Request) (interface{}, int, error) { ctx = setupHandlerContext(ctx, r) response, err := storageHandler.CommitWrite(ctx, r) + if err != nil { - return nil, err + + if errors.Is(common.ErrFileWasDeleted, err) { + return response, http.StatusNoContent, nil + } + return nil, http.StatusBadRequest, err } - return response, nil + return response, http.StatusOK, nil } func ReferencePathHandler(ctx context.Context, r *http.Request) (interface{}, error) { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 4fd456ff1..bddb45e4a 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -515,7 +515,9 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } err = connectionObj.CommitToFileStore(ctx) if err != nil { - return nil, common.NewError("file_store_error", "Error committing to file store. "+err.Error()) + if !errors.Is(common.ErrFileWasDeleted, err) { + return nil, common.NewError("file_store_error", "Error committing to file store. "+err.Error()) + } } result.Changes = connectionObj.Changes @@ -529,6 +531,9 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b result.Success = true result.ErrorMessage = "" + if errors.Is(common.ErrFileWasDeleted, err) { + return &result, err + } return &result, nil } diff --git a/code/go/0chain.net/core/common/constants.go b/code/go/0chain.net/core/common/constants.go index 2345aebe1..4314f134b 100644 --- a/code/go/0chain.net/core/common/constants.go +++ b/code/go/0chain.net/core/common/constants.go @@ -32,4 +32,7 @@ var ( // ErrEntityNotFound entity can't found in db ErrEntityNotFound = errors.New("entity not found") + + // ErrFileWasDeleted file already was deleted + ErrFileWasDeleted = errors.New("file was deleted") ) diff --git a/code/go/0chain.net/core/common/errors.go b/code/go/0chain.net/core/common/errors.go index 052696808..9f09e5b06 100644 --- a/code/go/0chain.net/core/common/errors.go +++ b/code/go/0chain.net/core/common/errors.go @@ -6,9 +6,8 @@ import ( /*Error type for a new application error */ type Error struct { - Code string `json:"code,omitempty"` - Msg string `json:"msg"` - StatusCode int `json:"status_code,omitempty"` + Code string `json:"code,omitempty"` + Msg string `json:"msg"` } func (err *Error) Error() string { @@ -25,11 +24,6 @@ func NewErrorf(code, format string, args ...interface{}) *Error { return &Error{Code: code, Msg: fmt.Sprintf(format, args...)} } -/*NewErrorf - create a new error with format */ -func NewErrorfWithStatusCode(statusCode int, errCode, format string, args ...interface{}) *Error { - return &Error{StatusCode: statusCode, Code: errCode, Msg: fmt.Sprintf(format, args...)} -} - /*InvalidRequest - create error messages that are needed when validating request input */ func InvalidRequest(msg string) error { return NewError("invalid_request", fmt.Sprintf("Invalid request (%v)", msg)) diff --git a/code/go/0chain.net/core/common/handler.go b/code/go/0chain.net/core/common/handler.go index 8f24c11b0..0dd0d4fcd 100644 --- a/code/go/0chain.net/core/common/handler.go +++ b/code/go/0chain.net/core/common/handler.go @@ -62,18 +62,14 @@ func ToByteStream(handler JSONResponderF) ReqRespHandlerf { ctx := r.Context() data, err := handler(ctx, r) if err != nil { - statusCode := 400 if cerr, ok := err.(*Error); ok { w.Header().Set(AppErrorHeader, cerr.Code) - if cerr.StatusCode != 0 { - statusCode = cerr.StatusCode - } } if data != nil { responseString, _ := json.Marshal(data) - http.Error(w, string(responseString), statusCode) + http.Error(w, string(responseString), 400) } else { - http.Error(w, err.Error(), statusCode) + http.Error(w, err.Error(), 400) } } else if data != nil { rawdata, ok := data.([]byte) @@ -154,3 +150,57 @@ func JSONString(json map[string]interface{}, field string, required bool) (strin return fmt.Sprintf("%v", sval), nil } } + +type StatusCodeResponderF func(ctx context.Context, r *http.Request) (interface{}, int, error) + +func ToStatusCode(handler StatusCodeResponderF) ReqRespHandlerf { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all. + if r.Method == "OPTIONS" { + SetupCORSResponse(w, r) + return + } + + ctx := r.Context() + + data, statusCode, err := handler(ctx, r) + + if err != nil { + if statusCode == 0 { + statusCode = http.StatusBadRequest + } + + w.WriteHeader(statusCode) + w.Header().Set("Content-Type", "application/json") + + if data != nil { + json.NewEncoder(w).Encode(data) //nolint:errcheck + } else { + //nolint:errcheck + json.NewEncoder(w).Encode(map[string]string{ + "error": err.Error(), + }) + } + + return + } + + if statusCode == 0 { + statusCode = http.StatusOK + } + + w.WriteHeader(statusCode) + + if data != nil { + + rawdata, ok := data.([]byte) + if ok { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(rawdata) //nolint:errcheck + } else { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(data) //nolint:errcheck + } + } + } +} From 03d61ae405c78789eb344906e71ec3a80fa4e3c4 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 2 Mar 2022 23:03:01 +0800 Subject: [PATCH 115/161] Feat/hashnode API (#558) * feat(writemarker): updated WriteMarker mutex * feat(writemarker): updated CreateAt with int64 * fix(ref): load root hash node * fix(ref): added unit tests for hashnode * fix(ref): added unit tests for hashnode * feat(writemarker): updated WriteMarker mutex * feat(writemarker): updated CreateAt with int64 * fix(ref): load root hash node * fix(ref): added unit tests for hashnode * fix(ref): added unit tests for hashnode * fix(ref):fixed typo * fix(ref):fixed typo * fix(writelock):renamed connectionID with sessionID * fix(writelock):renamed connectionID with sessionID * fix(writelock):revered sessionID to connectionID * fix(gomod): upgraded gorm * feat(writemarker): fixed sql issue on postgres * feat(gomod): updated gosdk --- .../0chain.net/blobbercore/handler/context.go | 6 +- .../0chain.net/blobbercore/handler/handler.go | 6 +- .../blobbercore/handler/handler_hashnode.go | 16 + .../handler/handler_hashnode_test.go | 113 +++++ .../handler/handler_writemarker.go | 6 +- .../handler/handler_writemarker_test.go | 4 +- code/go/0chain.net/blobbercore/mock/init.go | 8 +- .../0chain.net/blobbercore/reference/dao.go | 73 ---- .../blobbercore/reference/entity.go | 40 +- .../blobbercore/reference/hashnode.go | 64 +++ .../blobbercore/reference/hashnode_test.go | 390 ++++++++++++++++++ .../blobbercore/writemarker/mutex.go | 10 +- .../blobbercore/writemarker/mutext_test.go | 48 +-- code/go/0chain.net/core/common/constants.go | 5 + code/go/0chain.net/core/transaction/http.go | 8 +- go.mod | 4 +- go.sum | 8 +- 17 files changed, 651 insertions(+), 158 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/handler_hashnode.go create mode 100644 code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go delete mode 100644 code/go/0chain.net/blobbercore/reference/dao.go create mode 100644 code/go/0chain.net/blobbercore/reference/hashnode.go create mode 100644 code/go/0chain.net/blobbercore/reference/hashnode_test.go diff --git a/code/go/0chain.net/blobbercore/handler/context.go b/code/go/0chain.net/blobbercore/handler/context.go index 0f27146a5..7286d2779 100644 --- a/code/go/0chain.net/blobbercore/handler/context.go +++ b/code/go/0chain.net/blobbercore/handler/context.go @@ -84,8 +84,7 @@ func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.Re statusCode = http.StatusInternalServerError } - buf, _ := json.Marshal(err) - http.Error(w, string(buf), statusCode) + http.Error(w, err.Error(), statusCode) return } @@ -97,8 +96,7 @@ func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.Re statusCode = http.StatusInternalServerError } - buf, _ := json.Marshal(err) - http.Error(w, string(buf), statusCode) + http.Error(w, err.Error(), statusCode) return } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index cb7a2710d..29f0069b8 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -78,8 +78,10 @@ func SetupHandlers(r *mux.Router) { // lightweight http handler without heavy postgres transaction to improve performance - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost) - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost, http.MethodOptions) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) + + r.HandleFunc("/v1/hashnode/root/{allocation}", WithHandler(LoadRootHashnode)).Methods(http.MethodGet, http.MethodOptions) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/blobbercore/handler/handler_hashnode.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go new file mode 100644 index 000000000..2a916353a --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go @@ -0,0 +1,16 @@ +//go:build !integration_tests +// +build !integration_tests + +package handler + +import "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx *Context) (interface{}, error) { + + root, err := reference.LoadRootHashnode(ctx, ctx.AllocationTx) + if err != nil { + return nil, err + } + return root, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go new file mode 100644 index 000000000..1515b9878 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go @@ -0,0 +1,113 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/gorilla/mux" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashnodeHanders_LoadRootHashnode(t *testing.T) { + + datastore.UseMocket(true) + + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_handler_load_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + r := mux.NewRouter() + SetupHandlers(r) + + req, err := http.NewRequest(http.MethodGet, "/v1/refs/root/{allocation}", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(WithHandler(func(ctx *Context) (interface{}, error) { + ctx.AllocationTx = "allocation_handler_load_root" + return LoadRootHashnode(ctx) + })) + + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + + var root reference.Hashnode + + err = json.Unmarshal(rr.Body.Bytes(), &root) + require.Nil(t, err) + + require.NotNil(t, root) + require.Len(t, root.Children, 2) + + require.Equal(t, root.Children[0].Name, "sub1") + require.Len(t, root.Children[0].Children, 1) + require.Equal(t, root.Children[0].Children[0].Name, "file1") + require.Equal(t, root.Children[1].Name, "sub2") +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go index 02196fd5f..2338dde80 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go @@ -11,10 +11,10 @@ var WriteMarkerMutext = &writemarker.Mutex{} // LockWriteMarker try to lock writemarker for specified allocation id, and return latest RefTree func LockWriteMarker(ctx *Context) (interface{}, error) { - sessionID := ctx.FormValue("session_id") + connectionID := ctx.FormValue("connection_id") requestTime := ctx.FormTime("request_time") - result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationTx, sessionID, requestTime) + result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationTx, connectionID, requestTime) if err != nil { return nil, err } @@ -24,7 +24,7 @@ func LockWriteMarker(ctx *Context) (interface{}, error) { // UnlockWriteMarker release WriteMarkerMutex func UnlockWriteMarker(ctx *Context) (interface{}, error) { - sessionID := ctx.FormValue("session_id") + sessionID := ctx.FormValue("connection_id") err := WriteMarkerMutext.Unlock(ctx, ctx.AllocationTx, sessionID) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go index 48779982f..0e18064df 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go @@ -28,7 +28,7 @@ func TestWriteMarkerHandlers_Lock(t *testing.T) { now := time.Now() - formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("connection_id", "connection_id") //nolint: errcheck formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck formWriter.Close() @@ -68,7 +68,7 @@ func TestWriteMarkerHandlers_Unlock(t *testing.T) { now := time.Now() - formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("connection_id", "connection_id") //nolint: errcheck formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck formWriter.Close() diff --git a/code/go/0chain.net/blobbercore/mock/init.go b/code/go/0chain.net/blobbercore/mock/init.go index 2564584ad..e1c2fa978 100644 --- a/code/go/0chain.net/blobbercore/mock/init.go +++ b/code/go/0chain.net/blobbercore/mock/init.go @@ -1,10 +1,12 @@ package mock import ( + "encoding/json" "io" "net/http" "net/http/httptest" + "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/gosdk/sdks" "github.com/0chain/gosdk/sdks/blobber" ) @@ -18,11 +20,13 @@ const ( // ) func NewBlobberClient() *blobber.Blobber { - z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain") - err := z.InitWallet(zboxWallet) + wallet := &zcncrypto.Wallet{} + err := json.Unmarshal([]byte(zboxWallet), wallet) //nolint: errcheck if err != nil { panic("mock: z.InitWallet " + err.Error()) } + z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain", wallet) + z.NewRequest = func(method, url string, body io.Reader) (*http.Request, error) { return httptest.NewRequest(method, url, body), nil } diff --git a/code/go/0chain.net/blobbercore/reference/dao.go b/code/go/0chain.net/blobbercore/reference/dao.go deleted file mode 100644 index 363ce4825..000000000 --- a/code/go/0chain.net/blobbercore/reference/dao.go +++ /dev/null @@ -1,73 +0,0 @@ -package reference - -import ( - "context" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/errors" - "gorm.io/gorm" -) - -// LoadRootNode load root node with its descendant nodes -func LoadRootNode(ctx context.Context, allocationID string) (*HashNode, error) { - - db := datastore.GetStore().GetDB() - - db = db.Where("allocation_id = ? and deleted_at IS NULL", allocationID) - - db = db.Order("level desc, path") - - dict := make(map[string][]*HashNode) - - var nodes []*HashNode - // it is better to load them in batched if there are a lot of objects in db - err := db.FindInBatches(&nodes, 100, func(tx *gorm.DB, batch int) error { - // batch processing found records - for _, object := range nodes { - dict[object.ParentPath] = append(dict[object.ParentPath], object) - - for _, child := range dict[object.Path] { - object.AddChild(child) - } - } - - return nil - }).Error - - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - - // create empty dir if root is missing - if len(dict) == 0 { - return &HashNode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil - } - - rootNodes, ok := dict[""] - - if ok { - if len(rootNodes) == 1 { - return rootNodes[0], nil - } - - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") - } - - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") -} - -const ( - SQLWhereGetByAllocationTxAndPath = "reference_objects.allocation_id = ? and reference_objects.path = ? and deleted_at is NULL" -) - -// DryRun Creates a prepared statement when executing any SQL and caches them to speed up future calls -// https://gorm.io/docs/performance.html#Caches-Prepared-Statement -func DryRun(db *gorm.DB) { - - // https://gorm.io/docs/session.html#DryRun - // Session mode - //tx := db.Session(&gorm.Session{PrepareStmt: true, DryRun: true}) - - // use Table instead of Model to reduce reflect times -} diff --git a/code/go/0chain.net/blobbercore/reference/entity.go b/code/go/0chain.net/blobbercore/reference/entity.go index 2fae95c28..d0e2ee523 100644 --- a/code/go/0chain.net/blobbercore/reference/entity.go +++ b/code/go/0chain.net/blobbercore/reference/entity.go @@ -1,14 +1,11 @@ package reference import ( - "strconv" - "strings" - "gorm.io/datatypes" ) -// HashNode ref node in hash tree -type HashNode struct { +// Hashnode ref node in hash tree +type Hashnode struct { // hash data AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` Type string `gorm:"column:type" json:"type,omitempty"` @@ -24,46 +21,23 @@ type HashNode struct { // other data ParentPath string `gorm:"parent_path" json:"-"` - Children []*HashNode `gorm:"-" json:"children,omitempty"` + Children []*Hashnode `gorm:"-" json:"children,omitempty"` } // TableName get table name of Ref -func (HashNode) TableName() string { +func (Hashnode) TableName() string { return TableNameReferenceObjects } -func (n *HashNode) AddChild(c *HashNode) { +func (n *Hashnode) AddChild(c *Hashnode) { if n.Children == nil { - n.Children = make([]*HashNode, 0, 10) + n.Children = make([]*Hashnode, 0, 10) } n.Children = append(n.Children, c) } // GetLookupHash get lookuphash -func (n *HashNode) GetLookupHash() string { +func (n *Hashnode) GetLookupHash() string { return GetReferenceLookup(n.AllocationID, n.Path) } - -// GetHashCode get hash code -func (n *HashNode) GetHashCode() string { - - if len(n.Attributes) == 0 { - n.Attributes = datatypes.JSON("{}") - } - hashArray := []string{ - n.AllocationID, - n.Type, - n.Name, - n.Path, - strconv.FormatInt(n.Size, 10), - n.ContentHash, - n.MerkleRoot, - strconv.FormatInt(n.ActualFileSize, 10), - n.ActualFileHash, - string(n.Attributes), - strconv.FormatInt(n.ChunkSize, 10), - } - - return strings.Join(hashArray, ":") -} diff --git a/code/go/0chain.net/blobbercore/reference/hashnode.go b/code/go/0chain.net/blobbercore/reference/hashnode.go new file mode 100644 index 000000000..a2393e6bd --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hashnode.go @@ -0,0 +1,64 @@ +package reference + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/errors" +) + +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx context.Context, allocationID string) (*Hashnode, error) { + + db := datastore.GetStore().GetDB() + + db = db.Raw(` +SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects +WHERE allocation_id = ? and deleted_at IS NULL +ORDER BY level desc, path`, allocationID) + + rows, err := db.Rows() + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + defer rows.Close() + + nodes := make(map[string]*Hashnode) + for rows.Next() { + + node := &Hashnode{} + err = db.ScanRows(rows, node) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + _, ok := nodes[node.Path] + if ok { + return nil, common.ErrDuplicatedNode + } + + nodes[node.Path] = node + + parent, ok := nodes[node.ParentPath] + if ok { + parent.AddChild(node) + } + + } + + // create empty dir if root is missing + if len(nodes) == 0 { + return &Hashnode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil + } + + root, ok := nodes["/"] + + if ok { + return root, nil + } + + return nil, common.ErrMissingRootNode +} diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go new file mode 100644 index 000000000..c6a8d5ae5 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -0,0 +1,390 @@ +package reference + +import ( + "context" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashnode_Should_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "No any node should work", + allocationID: "allocation_none", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_none"). + WithReply(nil) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 0) + + require.Equal(test, allocationID, r.AllocationID) + require.Equal(test, DIRECTORY, r.Type) + require.Equal(test, "/", r.Name) + require.Equal(test, "/", r.Path) + require.Equal(test, "", r.ContentHash) + require.Equal(test, "", r.MerkleRoot) + require.Equal(test, "", r.ActualFileHash) + require.EqualValues(test, 0, r.ChunkSize) + require.EqualValues(test, 0, r.Size) + require.EqualValues(test, 0, r.ActualFileSize) + + buf, e := r.Attributes.MarshalJSON() //nolint + require.Nil(test, e) + + require.Equal(test, "null", string(buf)) + + require.Equal(test, "", r.ParentPath) + + }, + }, + { + name: "Nested node should work", + allocationID: "allocation_nested", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_nested"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 2) + + require.Equal(test, r.Children[0].Name, "sub1") + require.Len(test, r.Children[0].Children, 1) + require.Equal(test, r.Children[0].Children[0].Name, "file1") + require.Equal(test, r.Children[1].Name, "sub2") + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRootHashnode(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} + +func TestHashnode_Should_Not_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "Missing root node should not work", + allocationID: "allocation_missing", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_missing"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_missing", + "type": "D", + "name": "sub", + "path": "/sub", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }}) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrMissingRootNode, err) + + }, + }, + { + name: "Duplicated root node should not work", + allocationID: "allocation_duplicated_root", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + { + name: "Duplicated node should not work", + allocationID: "allocation_duplicated_node", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_node"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRootHashnode(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index daeaddab0..8c2349f92 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -67,7 +67,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req CreatedAt: *requestTime, } - err = db.Create(&lock).Error + err = db.Table(datastore.TableNameWriteLock).Create(&lock).Error if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } @@ -92,7 +92,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req lock.ConnectionID = connectionID lock.CreatedAt = *requestTime - err = db.Save(&lock).Error + err = db.Table(datastore.TableNameWriteLock).Where("allocation_id=?", allocationID).Save(&lock).Error if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } @@ -120,19 +120,19 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req } -func (*Mutex) Unlock(ctx context.Context, allocationID string, sessionID string) error { +func (*Mutex) Unlock(ctx context.Context, allocationID string, connectionID string) error { if len(allocationID) == 0 { return nil } - if len(sessionID) == 0 { + if len(connectionID) == 0 { return nil } db := datastore.GetStore().GetDB() - err := db.Where("allocation_id = ? and session_id =? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error + err := db.Where("allocation_id = ? and connection_id = ? ", allocationID, connectionID).Delete(&datastore.WriteLock{}).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index e36f0fe15..b7fb23810 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -61,6 +61,7 @@ func TestMutext_LockShouldWork(t *testing.T) { assert: func(test *testing.T, r *LockResult, err error) { require.Nil(test, err) require.Equal(test, LockStatusOK, r.Status) + require.EqualValues(test, now.Unix(), r.CreatedAt) }, }, { @@ -86,7 +87,7 @@ func TestMutext_LockShouldWork(t *testing.T) { }, }, { - name: "lock should ok if it is timeout", + name: "lock should be ok if it is timeout", allocationID: "lock_timeout_allocation_id", connectionID: "lock_timeout_2nd_connection_id", requestTime: now, @@ -107,6 +108,28 @@ func TestMutext_LockShouldWork(t *testing.T) { require.Equal(test, LockStatusPending, r.Status) }, }, + { + name: "retry lock by same request should work if it is timeout", + allocationID: "lock_same_timeout_allocation_id", + connectionID: "lock_same_timeout_connection_id", + requestTime: now, + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). + WithArgs("lock_same_timeout_allocation_id"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "lock_same_timeout_allocation_id", + "connection_id": "lock_same_timeout_connection_id", + "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), + }, + }) + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, err) + require.NotNil(test, r) + }, + }, } for _, it := range tests { @@ -134,7 +157,6 @@ func TestMutext_LockShouldNotWork(t *testing.T) { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second m := &Mutex{} - now := time.Now() tests := []struct { name string @@ -157,28 +179,6 @@ func TestMutext_LockShouldNotWork(t *testing.T) { require.NotNil(test, err) }, }, - { - name: "retry lock by same request should not work if it is timeout", - allocationID: "lock_same_timeout_allocation_id", - connectionID: "lock_same_timeout_connection_id", - requestTime: now, - mock: func() { - gomocket.Catcher.NewMock(). - WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). - WithArgs("lock_same_timeout_allocation_id"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "lock_same_timeout_allocation_id", - "connection_id": "lock_same_timeout_connection_id", - "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), - }, - }) - }, - assert: func(test *testing.T, r *LockResult, err error) { - require.NotNil(test, err) - require.Nil(test, r) - }, - }, } for _, it := range tests { diff --git a/code/go/0chain.net/core/common/constants.go b/code/go/0chain.net/core/common/constants.go index 4314f134b..8ed461d5c 100644 --- a/code/go/0chain.net/core/common/constants.go +++ b/code/go/0chain.net/core/common/constants.go @@ -33,6 +33,11 @@ var ( // ErrEntityNotFound entity can't found in db ErrEntityNotFound = errors.New("entity not found") + // ErrMissingRootNode root node is missing + ErrMissingRootNode = errors.New("root node is missing") + + // ErrDuplicatedNode duplicated nodes + ErrDuplicatedNode = errors.New("duplicated nodes") // ErrFileWasDeleted file already was deleted ErrFileWasDeleted = errors.New("file was deleted") ) diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index 57ba421de..dee99d909 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -1,6 +1,7 @@ package transaction import ( + "bytes" "context" "encoding/hex" "hash/fnv" @@ -107,7 +108,7 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] //leave first item for ErrTooLessConfirmation var msgList = make([]string, 1, numSharders) - r := resty.New(transport, func(req *http.Request, resp *http.Response, cancelFunc context.CancelFunc, err error) error { + r := resty.New(transport, func(req *http.Request, resp *http.Response, respBody []byte, cancelFunc context.CancelFunc, err error) error { if err != nil { //network issue msgList = append(msgList, err.Error()) return err @@ -116,7 +117,6 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] url := req.URL.String() if resp.StatusCode != http.StatusOK { - resp.Body.Close() errorMsg := "[sharder]" + resp.Status + ": " + url msgList = append(msgList, errorMsg) @@ -124,9 +124,9 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] } hash := fnv.New32() //use fnv for better performance - teeReader := io.TeeReader(resp.Body, hash) + + teeReader := io.TeeReader(bytes.NewReader(respBody), hash) resBody, err := io.ReadAll(teeReader) - resp.Body.Close() if err != nil { errorMsg := "[sharder]body: " + url + " " + err.Error() diff --git a/go.mod b/go.mod index d9e844bef..54279efc6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 + github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-ini/ini v1.55.0 // indirect @@ -38,7 +38,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.3.1 - gorm.io/gorm v1.23.1 + gorm.io/gorm v1.23.2 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index a154c6548..3c8684a81 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 h1:cOqg66kR646dc1NNgduE0Ridlg0bDDvhxOPC9rhrNWk= -github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= +github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da h1:Mno3gifncwxsLEfkGUnsKqTvCDUbUioJXkYYH8Thw+8= +github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -878,7 +878,6 @@ github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= @@ -1638,8 +1637,9 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.23.1 h1:aj5IlhDzEPsoIyOPtTRVI+SyaN1u6k613sbt4pwbxG0= gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.2 h1:xmq9QRMWL8HTJyhAUBXy8FqIIQCYESeKfJL4DoGKiWQ= +gorm.io/gorm v1.23.2/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From aef63d1b5fc77de9f4dd290ccd9d6945c47ff74d Mon Sep 17 00:00:00 2001 From: prince Date: Thu, 3 Mar 2022 04:30:36 +0530 Subject: [PATCH 116/161] skip minio setup if minio.start is false in config. (#564) Signed-off-by: Pradip Parmar Co-authored-by: Pradip Parmar --- code/go/0chain.net/blobber/minio.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/code/go/0chain.net/blobber/minio.go b/code/go/0chain.net/blobber/minio.go index 1d5140fa1..d62586abf 100644 --- a/code/go/0chain.net/blobber/minio.go +++ b/code/go/0chain.net/blobber/minio.go @@ -5,12 +5,19 @@ import ( "fmt" "os" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) func setupMinio() error { fmt.Print("[4/12] setup minio") + + if config.Configuration.MinioStart { + fmt.Print(" + No minio [SKIP]\n") + return nil + } + reader, err := os.Open(minioFile) if err != nil { return err From f969d5ef073802864e0c7b596fbd7ebac08940d6 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 5 Mar 2022 11:26:38 +0800 Subject: [PATCH 117/161] fix(devops): enabled build job for tag (#566) * fix(devops): enabled build job for tag * fix(devops): enabled build job for tag * Update build-&-publish-docker-image.yml --- .github/workflows/build-&-publish-docker-image.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index ae879038c..71bb8a4ce 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -9,11 +9,9 @@ on: branches: - staging tags: + - '*' pull_request: - workflow_dispatch: - - env: BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} @@ -202,4 +200,4 @@ jobs: context: "0Chain System Tests" target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} env: - GITHUB_TOKEN: ${{ github.token }} \ No newline at end of file + GITHUB_TOKEN: ${{ github.token }} From 0d4a61ae48850d5dd7976569fc15f4d1b2b59a76 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 5 Mar 2022 16:00:56 +0800 Subject: [PATCH 118/161] fix(gomod): upgraded gprc-gateway with v2.8.0 (#567) --- go.mod | 5 ++--- go.sum | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 54279efc6..ae3b4c261 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.8.0 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.15.0 github.com/jackc/pgx/v4 v4.15.0 // indirect @@ -27,10 +27,9 @@ require ( go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.21.0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068 + google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a google.golang.org/grpc v1.44.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index 3c8684a81..b4b537fa2 100644 --- a/go.sum +++ b/go.sum @@ -460,8 +460,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 h1:I8MsauTJQXZ8df8qJvEln0kYNc3bSapuaSsEsnFdEFU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3/go.mod h1:lZdb/YAJUSj9OqrCHs2ihjtoO3+xK3G53wTYXFWRGDo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.8.0 h1:/57CcG6YAO31FWwTVUnNl+2RcDUAaT55HtI71KQAXVw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.8.0/go.mod h1:/fckq3NE+vGiJsd4fDt4ge1XrK8cN+e5G5QWIzdg7Q8= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -1201,6 +1201,7 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1524,9 +1525,8 @@ google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068 h1:pwzFiZfBTH/GjBWz1BcDwMBaHBo8mZvpLa7eBKJpFAk= -google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a h1:uqouglH745GoGeZ1YFZbPBiu961tgi/9Qm5jaorajjQ= +google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= From 7602616510946195013a9dcbaaca0d0bdf4e34a3 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 5 Mar 2022 20:04:58 +0800 Subject: [PATCH 119/161] fix(writelock):release lock if file is committed (#570) * fix(writelock):release lock if file is committed * fix(writelock):improved delete handler with restful delete method * fix(handler): fixed signature issue for collaborator --- .../allocation/file_changer_base.go | 45 ++++++++++--------- .../allocation/file_changer_update.go | 2 + .../0chain.net/blobbercore/handler/context.go | 32 +++++++++++-- .../0chain.net/blobbercore/handler/handler.go | 2 +- .../handler/handler_writemarker.go | 4 +- .../handler/handler_writemarker_test.go | 3 +- .../blobbercore/writemarker/mutex.go | 2 +- 7 files changed, 62 insertions(+), 28 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 4f0544250..b566117d6 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -3,6 +3,7 @@ package allocation import ( "context" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -53,40 +54,44 @@ type BaseFileChanger struct { UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to } -func (nf *BaseFileChanger) DeleteTempFile() error { +func (fc *BaseFileChanger) DeleteTempFile() error { fileInputData := &filestore.FileInputData{} - fileInputData.Name = nf.Filename - fileInputData.Path = nf.Path - fileInputData.Hash = nf.Hash - err := filestore.GetFileStore().DeleteTempFile(nf.AllocationID, fileInputData, nf.ConnectionID) - if nf.ThumbnailSize > 0 { + fileInputData.Name = fc.Filename + fileInputData.Path = fc.Path + fileInputData.Hash = fc.Hash + err := filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fileInputData, fc.ConnectionID) + if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} - fileInputData.Name = nf.ThumbnailFilename - fileInputData.Path = nf.Path - fileInputData.Hash = nf.ThumbnailHash - err = filestore.GetFileStore().DeleteTempFile(nf.AllocationID, fileInputData, nf.ConnectionID) + fileInputData.Name = fc.ThumbnailFilename + fileInputData.Path = fc.Path + fileInputData.Hash = fc.ThumbnailHash + err = filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fileInputData, fc.ConnectionID) } return err } -func (nfch *BaseFileChanger) CommitToFileStore(ctx context.Context) error { +func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error { fileInputData := &filestore.FileInputData{} - fileInputData.Name = nfch.Filename - fileInputData.Path = nfch.Path - fileInputData.Hash = nfch.Hash - _, err := filestore.GetFileStore().CommitWrite(nfch.AllocationID, fileInputData, nfch.ConnectionID) + fileInputData.Name = fc.Filename + fileInputData.Path = fc.Path + fileInputData.Hash = fc.Hash + _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fileInputData, fc.ConnectionID) if err != nil { return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) } - if nfch.ThumbnailSize > 0 { + if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} - fileInputData.Name = nfch.ThumbnailFilename - fileInputData.Path = nfch.Path - fileInputData.Hash = nfch.ThumbnailHash - _, err := filestore.GetFileStore().CommitWrite(nfch.AllocationID, fileInputData, nfch.ConnectionID) + fileInputData.Name = fc.ThumbnailFilename + fileInputData.Path = fc.Path + fileInputData.Hash = fc.ThumbnailHash + _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fileInputData, fc.ConnectionID) if err != nil { return common.NewError("file_store_error", "Error committing thumbnail to file store. "+err.Error()) } } + + // release WriteMarkerMutex + datastore.GetStore().GetTransaction(ctx).Exec("DELETE FROM write_locks WHERE allocation_id = ? and connection_id = ? ", fc.AllocationID, fc.ConnectionID) + return nil } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 1af88e223..c1568e48c 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -10,6 +10,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" + "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" @@ -108,6 +109,7 @@ func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error { } } } + return nf.BaseFileChanger.CommitToFileStore(ctx) } diff --git a/code/go/0chain.net/blobbercore/handler/context.go b/code/go/0chain.net/blobbercore/handler/context.go index 7286d2779..31cc9e561 100644 --- a/code/go/0chain.net/blobbercore/handler/context.go +++ b/code/go/0chain.net/blobbercore/handler/context.go @@ -30,19 +30,36 @@ type Context struct { Allocation *allocation.Allocation + // Vars route variables + Vars map[string]string + Store datastore.Store Request *http.Request StatusCode int } +func (c *Context) Var(key string) string { + if c == nil || c.Vars == nil { + return "" + } + + return c.Vars[key] +} + // FormValue get value from form data func (c *Context) FormValue(key string) string { + if c == nil || c.Vars == nil { + return "" + } return c.Request.FormValue(key) } // FormTime get time from form data func (c *Context) FormTime(key string) *time.Time { + if c == nil || c.Vars == nil { + return nil + } value := c.Request.FormValue(key) if len(value) == 0 { return nil @@ -121,11 +138,14 @@ func WithAuth(r *http.Request) (*Context, error) { Store: datastore.GetStore(), } - var vars = mux.Vars(r) + ctx.Vars = mux.Vars(r) + if ctx.Vars == nil { + ctx.Vars = make(map[string]string) + } ctx.ClientID = r.Header.Get(common.ClientHeader) ctx.ClientKey = r.Header.Get(common.ClientKeyHeader) - ctx.AllocationTx = vars["allocation"] + ctx.AllocationTx = ctx.Vars["allocation"] ctx.Signature = r.Header.Get(common.ClientSignatureHeader) if len(ctx.AllocationTx) > 0 { @@ -144,7 +164,13 @@ func WithAuth(r *http.Request) (*Context, error) { ctx.Allocation = alloc - valid, err := verifySignatureFromRequest(ctx.AllocationTx, ctx.Signature, alloc.OwnerPublicKey) + publicKey := alloc.OwnerPublicKey + + // request by collaborator + if alloc.OwnerID != ctx.ClientID { + publicKey = ctx.ClientKey + } + valid, err := verifySignatureFromRequest(ctx.AllocationTx, ctx.Signature, publicKey) if !valid { ctx.StatusCode = http.StatusBadRequest diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 29f0069b8..33b2a19fb 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -79,7 +79,7 @@ func SetupHandlers(r *mux.Router) { // lightweight http handler without heavy postgres transaction to improve performance r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost, http.MethodOptions) - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) + r.HandleFunc("/v1/writemarker/lock/{allocation}/{connection}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) r.HandleFunc("/v1/hashnode/root/{allocation}", WithHandler(LoadRootHashnode)).Methods(http.MethodGet, http.MethodOptions) } diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go index 2338dde80..c3d5227d1 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go @@ -24,9 +24,9 @@ func LockWriteMarker(ctx *Context) (interface{}, error) { // UnlockWriteMarker release WriteMarkerMutex func UnlockWriteMarker(ctx *Context) (interface{}, error) { - sessionID := ctx.FormValue("connection_id") + connectionID := ctx.Vars["connection"] - err := WriteMarkerMutext.Unlock(ctx, ctx.AllocationTx, sessionID) + err := WriteMarkerMutext.Unlock(ctx, ctx.AllocationTx, connectionID) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go index 0e18064df..9aefe3388 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go @@ -72,7 +72,7 @@ func TestWriteMarkerHandlers_Unlock(t *testing.T) { formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck formWriter.Close() - req, err := http.NewRequest(http.MethodDelete, "/v1/writemarker/lock/{allocation}", body) + req, err := http.NewRequest(http.MethodDelete, "/v1/writemarker/lock/{allocation}/{connection}", body) if err != nil { t.Fatal(err) } @@ -82,6 +82,7 @@ func TestWriteMarkerHandlers_Unlock(t *testing.T) { rr := httptest.NewRecorder() handler := http.HandlerFunc(WithHandler(func(ctx *Context) (interface{}, error) { ctx.AllocationTx = "TestHandlers_Unlock_allocation_id" + ctx.Vars["connection"] = "connection_id" return UnlockWriteMarker(ctx) })) diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 8c2349f92..4cdd04233 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -132,7 +132,7 @@ func (*Mutex) Unlock(ctx context.Context, allocationID string, connectionID stri db := datastore.GetStore().GetDB() - err := db.Where("allocation_id = ? and connection_id = ? ", allocationID, connectionID).Delete(&datastore.WriteLock{}).Error + err := db.Exec("DELETE FROM write_locks WHERE allocation_id = ? and connection_id = ? ", allocationID, connectionID).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil From 9e894f5f199cc66407528979bbf5c24f8994dd6b Mon Sep 17 00:00:00 2001 From: Lz Date: Sun, 6 Mar 2022 13:28:05 +0800 Subject: [PATCH 120/161] fix(devops): build multi platform images (#568) --- .../build-&-publish-docker-image.yml | 64 ++++++++----------- .gitignore | 1 + docker.local/base.Dockerfile | 33 ++++++---- docker.local/bin/build.base.sh | 25 +++++++- docker.local/bin/build.blobber.sh | 35 ++++++---- docker.local/bin/build.validator.sh | 35 ++++++---- docker.local/blobber.Dockerfile | 4 +- docker.local/validator.Dockerfile | 10 +-- 8 files changed, 121 insertions(+), 86 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 71bb8a4ce..333cf9287 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -31,43 +31,36 @@ jobs: echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV echo "SHA=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}')" >> $GITHUB_ENV - + - name: Setup go 1.17 uses: actions/setup-go@v2 with: go-version: '1.17' # The Go version to download (if necessary) and use. - # - uses: actions/cache@v2 - # with: - # path: ~/go/pkg/mod - # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - # restore-keys: | - # ${{ runner.os }}-go- - - - name: Clone blobber uses: actions/checkout@v1 - - name: Build blobber_base - run: ./docker.local/bin/build.base.sh - - - name: Build blobber - run: ./docker.local/bin/build.blobber.sh - - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push blobber + - name: Build base run: | - docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG - docker push ${BLOBBER_REGISTRY}:$TAG || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG; } + export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + ./docker.local/bin/build.base.sh + - name: Build blobber + run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) - docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA - docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA; } + + export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" + export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + + ./docker.local/bin/build.blobber.sh validator: runs-on: [self-hosted, build] @@ -89,36 +82,31 @@ jobs: with: go-version: '1.17' # The Go version to download (if necessary) and use. - # - uses: actions/cache@v2 - # with: - # path: ~/go/pkg/mod - # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - # restore-keys: | - # ${{ runner.os }}-go- - - name: Clone blobber uses: actions/checkout@v1 - - name: Build blobber_base - run: ./docker.local/bin/build.base.sh - - - name: Build validator - run: ./docker.local/bin/build.validator.sh - - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push validator + - name: Build base run: | - docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG - docker push ${VALIDATOR_REGISTRY}:$TAG || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG; } + export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + ./docker.local/bin/build.base.sh + - name: Build validator + run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) - docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA - docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA; } + + export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" + export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + + ./docker.local/bin/build.validator.sh + system-tests: if: github.event_name != 'workflow_dispatch' diff --git a/.gitignore b/.gitignore index 3ba7a8d29..152b4072b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ __debug_bin dev.local/data out/ **/tmp/ +*.tar.gz diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 583b31c20..b9c562b5b 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -2,6 +2,7 @@ FROM golang:1.17.1-alpine3.14 as blobber_base LABEL zchain="blobber" +# https://mirrors.alpinelinux.org/ # RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories @@ -16,21 +17,25 @@ RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/reposit RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories -RUN echo "https://mirror.yandex.ru/alpine/v3.14/main" >> /etc/apk/repositories -RUN echo "https://mirror.yandex.ru/alpine/v3.14/community" >> /etc/apk/repositories -RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep +RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep gmp gmp-dev openssl-dev # Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev && \ - cd /tmp && \ - wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ - wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && \ - mv bls* bls && \ - make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib && \ - make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \ - rm -R /tmp/mcl && \ - rm -R /tmp/bls +WORKDIR /tmp + +COPY ./docker.local/bin/mcl.tar.gz ./ +COPY ./docker.local/bin/bls.tar.gz ./ + +RUN tar zxvf mcl.tar.gz && rm mcl.tar.gz && mv mcl* mcl + +RUN tar zxvf bls.tar.gz && rm bls.tar.gz && mv bls* bls + +RUN make -C mcl -j $(nproc) lib/libmclbn256.so install +RUN cp mcl/lib/libmclbn256.so /usr/local/lib + +RUN make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install + +RUN rm -R /tmp/mcl && rm -R /tmp/bls + +#ENV GOPROXY=https://goproxy.cn diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 252909943..1e813fa5e 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -3,6 +3,29 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT +echo "1> set DOCKER_IMAGE & DOCKER_BUILD" +if [ -z "$DOCKER_BUILD" ]; then + if [ "x86_64" != "$(uname -m)" ]; then + #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + DOCKER_BUILD="buildx build --platform linux/arm64" + else + DOCKER_BUILD="build" + fi +fi +if [ -z "$DOCKER_IMAGE_BASE" ]; then + DOCKER_IMAGE_BASE="blobber_base" +fi +echo " DOCKER_BUILD=$DOCKER_BUILD" +echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" -docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base \ No newline at end of file +echo "" +echo "2> download herumi" + +[ ! -f ./docker.local/bin/mcl.tar.gz ] && wget -O ./docker.local/bin/mcl.tar.gz https://github.com/herumi/mcl/archive/refs/tags/v1.57.tar.gz + +[ ! -f ./docker.local/bin/bls.tar.gz ] && wget -O ./docker.local/bin/bls.tar.gz https://github.com/herumi/bls/archive/refs/tags/v1.22.tar.gz + +echo "" +echo "3> docker build" +DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t $DOCKER_IMAGE_BASE \ No newline at end of file diff --git a/docker.local/bin/build.blobber.sh b/docker.local/bin/build.blobber.sh index 64daed5d1..93f4b36dc 100755 --- a/docker.local/bin/build.blobber.sh +++ b/docker.local/bin/build.blobber.sh @@ -4,22 +4,29 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT -cmd="build" +echo "1> set DOCKER_IMAGE & DOCKER_BUILD" +if [ -z "$DOCKER_BUILD" ]; then + if [ "x86_64" != "$(uname -m)" ]; then + #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + DOCKER_BUILD="buildx build --platform linux/arm64" + else + DOCKER_BUILD="build" + fi +fi -for arg in "$@" -do - case $arg in - -m1|--m1|m1) - echo "The build will be performed for Apple M1 chip" - cmd="buildx build --platform linux/amd64" - shift - ;; - esac -done +if [ -z "$DOCKER_IMAGE_BASE" ]; then + DOCKER_IMAGE_BASE="blobber_base" +fi -# [ -d ./gosdk ] && rm -rf gosdk -# cp -r ../gosdk ./ +if [ -z "$DOCKER_IMAGE_BLOBBER" ]; then + DOCKER_IMAGE_BLOBBER="-t blobber" +fi +echo " DOCKER_BUILD=$DOCKER_BUILD" +echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" +echo " DOCKER_IMAGE_BLOBBER=$DOCKER_IMAGE_BLOBBER" -docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/blobber.Dockerfile . -t blobber +echo "" +echo "2> docker build blobber" +DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/blobber.Dockerfile . $DOCKER_IMAGE_BLOBBER \ No newline at end of file diff --git a/docker.local/bin/build.validator.sh b/docker.local/bin/build.validator.sh index 162828098..e91b1b8c4 100755 --- a/docker.local/bin/build.validator.sh +++ b/docker.local/bin/build.validator.sh @@ -4,21 +4,28 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT -cmd="build" +echo "1> set DOCKER_IMAGE & DOCKER_BUILD" +if [ -z "$DOCKER_BUILD" ]; then + if [ "x86_64" != "$(uname -m)" ]; then + #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + DOCKER_BUILD="buildx build --platform linux/arm64" + else + DOCKER_BUILD="build" + fi +fi -for arg in "$@" -do - case $arg in - -m1|--m1|m1) - echo "The build will be performed for Apple M1 chip" - cmd="buildx build --platform linux/amd64" - shift - ;; - esac -done +if [ -z "$DOCKER_IMAGE_BASE" ]; then + DOCKER_IMAGE_BASE="blobber_base" +fi -# [ -d ./gosdk ] && rm -rf gosdk -# cp -r ../gosdk ./ +if [ -z "$DOCKER_IMAGE_VALIDATOR" ]; then + DOCKER_IMAGE_VALIDATOR="-t validator" +fi -docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/validator.Dockerfile . -t validator +echo " DOCKER_BUILD=$DOCKER_BUILD" +echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" +echo " DOCKER_IMAGE_VALIDATOR=$DOCKER_IMAGE_VALIDATOR" +echo "" +echo "2> docker build validator" +DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/validator.Dockerfile . $DOCKER_IMAGE_VALIDATOR \ No newline at end of file diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index db01b09fa..86409499d 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -1,4 +1,6 @@ -FROM blobber_base as blobber_build +# syntax=docker/dockerfile:1 +ARG DOCKER_IMAGE_BASE +FROM $DOCKER_IMAGE_BASE as blobber_build LABEL zchain="blobber" ENV SRC_DIR=/0chain diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index cc3f284c4..a84ad4184 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -1,9 +1,11 @@ -FROM blobber_base as validator_build +# syntax=docker/dockerfile:1 +ARG DOCKER_IMAGE_BASE +FROM $DOCKER_IMAGE_BASE as validator_build LABEL zchain="validator" -ENV SRC_DIR=/blobber +ENV SRC_DIR=/0chain ENV GO111MODULE=on #ENV GOPROXY=https://goproxy.cn,direct @@ -25,6 +27,6 @@ RUN apk add gmp gmp-dev openssl-dev COPY --from=validator_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ /usr/local/lib/ -ENV APP_DIR=/blobber +ENV APP_DIR=/validator WORKDIR $APP_DIR -COPY --from=validator_build $APP_DIR/code/go/0chain.net/validator/validator $APP_DIR/bin/validator \ No newline at end of file +COPY --from=validator_build /0chain/code/go/0chain.net/validator/validator $APP_DIR/bin/validator \ No newline at end of file From a895c787546de280ad2d9b907f54e5c8b842470d Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 9 Mar 2022 12:14:16 +0800 Subject: [PATCH 121/161] fix(challenge): removed unused lock on challenge worker (#574) --- .../blobbercore/challenge/challenge.go | 9 +---- .../0chain.net/blobbercore/handler/worker.go | 40 +++++++++++-------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index 23f171d0f..4a00b6f4f 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -10,7 +10,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "github.com/remeh/sizedwaitgroup" @@ -142,9 +141,6 @@ func processAccepted(ctx context.Context) { // loadValidationTickets load validation tickets for challenge func loadValidationTickets(ctx context.Context, challengeObj *ChallengeEntity) error { - mutex := lock.GetMutex(challengeObj.TableName(), challengeObj.ChallengeID) - mutex.Lock() - defer func() { if r := recover(); r != nil { logging.Logger.Error("[recover]LoadValidationTickets", zap.Any("err", r)) @@ -179,8 +175,7 @@ func commitProcessed(ctx context.Context) { if err := openchallenge.UnmarshalFields(); err != nil { logging.Logger.Error("ChallengeEntity_UnmarshalFields", zap.String("challenge_id", openchallenge.ChallengeID), zap.Error(err)) } - mutex := lock.GetMutex(openchallenge.TableName(), openchallenge.ChallengeID) - mutex.Lock() + redeemCtx := datastore.GetStore().CreateTransaction(ctx) err := openchallenge.CommitChallenge(redeemCtx, false) if err != nil { @@ -188,7 +183,7 @@ func commitProcessed(ctx context.Context) { zap.Error(err), zap.String("challenge_id", openchallenge.ChallengeID)) } - mutex.Unlock() + db := datastore.GetStore().GetTransaction(redeemCtx) db.Commit() if err == nil && openchallenge.Status == Committed { diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index cb161e864..1a4692722 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -10,6 +10,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/core/lock" + "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -31,27 +32,32 @@ func CleanupDiskFiles(ctx context.Context) error { var allocations []allocation.Allocation db.Find(&allocations) for _, allocationObj := range allocations { - mutex := lock.GetMutex(allocationObj.TableName(), allocationObj.ID) - mutex.Lock() - _ = filestore.GetFileStore().IterateObjects(allocationObj.ID, func(contentHash string, contentSize int64) { - var refs []reference.Ref - err := db.Table((reference.Ref{}).TableName()).Where(reference.Ref{ContentHash: contentHash, Type: reference.FILE}).Or(reference.Ref{ThumbnailHash: contentHash, Type: reference.FILE}).Find(&refs).Error - if err != nil { - Logger.Error("Error in cleanup of disk files.", zap.Error(err)) - return - } - if len(refs) == 0 { - Logger.Info("hash has no references. Deleting from disk", zap.Any("count", len(refs)), zap.String("hash", contentHash)) - if err := filestore.GetFileStore().DeleteFile(allocationObj.ID, contentHash); err != nil { - Logger.Error("FileStore_DeleteFile", zap.String("content_hash", contentHash), zap.Error(err)) - } - } - }) - mutex.Unlock() + cleanupAllocationFiles(db, &allocationObj) } return nil } +func cleanupAllocationFiles(db *gorm.DB, allocationObj *allocation.Allocation) { + mutex := lock.GetMutex(allocationObj.TableName(), allocationObj.ID) + mutex.Lock() + defer mutex.Unlock() + _ = filestore.GetFileStore().IterateObjects(allocationObj.ID, func(contentHash string, contentSize int64) { + var refs []reference.Ref + err := db.Table((reference.Ref{}).TableName()).Where(reference.Ref{ContentHash: contentHash, Type: reference.FILE}).Or(reference.Ref{ThumbnailHash: contentHash, Type: reference.FILE}).Find(&refs).Error + if err != nil { + Logger.Error("Error in cleanup of disk files.", zap.Error(err)) + return + } + if len(refs) == 0 { + Logger.Info("hash has no references. Deleting from disk", zap.Any("count", len(refs)), zap.String("hash", contentHash)) + if err := filestore.GetFileStore().DeleteFile(allocationObj.ID, contentHash); err != nil { + Logger.Error("FileStore_DeleteFile", zap.String("content_hash", contentHash), zap.Error(err)) + } + } + }) + +} + func cleanupTempFiles(ctx context.Context) { defer func() { if r := recover(); r != nil { From 90f324c0f30515b73ba4ce0ee3644292cd3ad627 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Thu, 10 Mar 2022 10:20:22 +0545 Subject: [PATCH 122/161] Fix file upload inside file (#578) * Reject file upload inside file * Reject directory creation inside filepath * Comment out to test issue * Fix issue due to sql query * Validate path to copy object * Optimize changes * Fix path value --- .../blobbercore/handler/file_command.go | 28 +++++++++++ .../blobbercore/handler/file_command_add.go | 19 ++++++- .../blobbercore/handler/handler_test.go | 28 +++++++---- .../handler/object_operation_handler.go | 35 +++++++++++-- .../handler/upload_integration_test.go | 5 +- .../0chain.net/blobbercore/reference/ref.go | 49 +++++++++++++++++++ code/go/0chain.net/core/common/utils.go | 26 +++++++++- 7 files changed, 173 insertions(+), 17 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/file_command.go b/code/go/0chain.net/blobbercore/handler/file_command.go index 08ed1e076..3ea8bae7b 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command.go +++ b/code/go/0chain.net/blobbercore/handler/file_command.go @@ -2,10 +2,14 @@ package handler import ( "context" + "fmt" "net/http" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) // FileCommand execute command for a file operation @@ -37,3 +41,27 @@ func createFileCommand(req *http.Request) FileCommand { return &AddFileCommand{} } } + +// validateParentPathType validates against any parent path not being directory. +func validateParentPathType(ctx context.Context, allocationID, fPath string) error { + paths, err := common.GetParentPaths(fPath) + if err != nil { + return err + } + + refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) + if err != nil { + logging.Logger.Error(err.Error()) + return common.NewError("database_error", "Got error while getting parent refs") + } + + for _, ref := range refs { + if ref == nil { + continue + } + if ref.Type == reference.FILE { + return common.NewError("invalid_path", fmt.Sprintf("parent path %v is of file type", ref.Path)) + } + } + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_add.go b/code/go/0chain.net/blobbercore/handler/file_command_add.go index 94c69a251..85933f64e 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_add.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_add.go @@ -3,7 +3,9 @@ package handler import ( "context" "encoding/json" + "fmt" "net/http" + "path/filepath" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" @@ -36,12 +38,25 @@ func (cmd *AddFileCommand) IsAuthorized(ctx context.Context, req *http.Request, return common.NewError("invalid_parameters", "Invalid parameters. Error parsing the meta data for upload."+err.Error()) } - exisitingFileRef, _ := reference.GetReference(ctx, allocationObj.ID, fileChanger.Path) - if exisitingFileRef != nil { + if !filepath.IsAbs(fileChanger.Path) { + return common.NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fileChanger.Path)) + } + + isExist, err := reference.IsRefExist(ctx, allocationObj.ID, fileChanger.Path) + if err != nil { + logging.Logger.Error(err.Error()) + return common.NewError("database_error", "Got db error while getting ref") + } + + if isExist { return common.NewError("duplicate_file", "File at path already exists") } + if err := validateParentPathType(ctx, allocationObj.ID, fileChanger.Path); err != nil { + return err + } + //create a FixedMerkleTree instance first, it will be reloaded from db in cmd.reloadChange if it is not first chunk //cmd.fileChanger.FixedMerkleTree = &util.FixedMerkleTree{} diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index f09bd17d8..dd092f71e 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -3,7 +3,6 @@ package handler import ( "bytes" "encoding/json" - "errors" "fmt" "io" "mime/multipart" @@ -864,7 +863,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q.Set("path", path) q.Set("new_name", newName) q.Set("connection_id", connectionID) - q.Set("dest", "dest") + q.Set("dest", "/dest") url.RawQuery = q.Encode() r, err := http.NewRequest(http.MethodPost, url.String(), nil) @@ -915,13 +914,15 @@ func TestHandlers_Requiring_Signature(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). WithArgs(alloc.ID, lookUpHash). WillReturnRows( - sqlmock.NewRows([]string{"type"}). - AddRow(reference.FILE), + sqlmock.NewRows([]string{"type", "name"}). + AddRow(reference.FILE, "path"), ) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(aa, aa). - WillReturnError(errors.New("")) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT "path","type" FROM "reference_objects" WHERE`)). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type"}). + AddRow("/dest", reference.DIRECTORY), + ) mock.ExpectExec(`INSERT INTO "allocation_connections"`). WithArgs(aa, aa, aa, aa, aa, aa, aa). @@ -1034,7 +1035,9 @@ func TestHandlers_Requiring_Signature(t *testing.T) { } q := url.Query() - formFieldByt, err := json.Marshal(&allocation.UpdateFileChanger{}) + formFieldByt, err := json.Marshal( + &allocation.UpdateFileChanger{ + BaseFileChanger: allocation.BaseFileChanger{Path: path}}) if err != nil { t.Fatal(err) } @@ -1110,9 +1113,16 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ) mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects"`)). - WithArgs(aa). + WithArgs(aa, aa). WillReturnError(gorm.ErrRecordNotFound) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(*) FROM "reference_objects"`)). + WithArgs(aa, aa). + WillReturnRows( + sqlmock.NewRows([]string{"count"}). + AddRow(0), + ) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocation_connections" WHERE`)). WithArgs(connectionID, alloc.ID, alloc.OwnerID, allocation.DeletedConnection). WillReturnRows( diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index bddb45e4a..85c0c6fcc 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" @@ -790,10 +791,30 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } + newPath := filepath.Join(destPath, objectRef.Name) - destRef, _ := reference.GetReference(ctx, allocationID, newPath) - if destRef != nil { - return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") + paths, err := common.GetParentPaths(newPath) + if err != nil { + return nil, err + } + + paths = append(paths, newPath) + + refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) + if err != nil { + Logger.Error("Database error", zap.Error(err)) + return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) + } + + for _, ref := range refs { + switch ref.Path { + case newPath: + return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") + default: + if ref.Type == reference.FILE { + return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) + } + } } allocationChange := &allocation.AllocationChange{} @@ -880,6 +901,10 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("invalid_parameters", "Invalid dir path passed") } + if !filepath.IsAbs(dirPath) { + return nil, common.NewError("invalid_path", fmt.Sprintf("%v is not absolute path", dirPath)) + } + exisitingRef := fsh.checkIfFileAlreadyExists(ctx, allocationID, dirPath) if allocationObj.OwnerID != clientID && allocationObj.PayerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") @@ -889,6 +914,10 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("duplicate_file", "File at path already exists") } + if err := validateParentPathType(ctx, allocationID, dirPath); err != nil { + return nil, err + } + connectionID := r.FormValue("connection_id") if connectionID == "" { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") diff --git a/code/go/0chain.net/blobbercore/handler/upload_integration_test.go b/code/go/0chain.net/blobbercore/handler/upload_integration_test.go index b06d7c048..1e938a373 100644 --- a/code/go/0chain.net/blobbercore/handler/upload_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/upload_integration_test.go @@ -25,7 +25,8 @@ func TestBlobberGRPCService_UploadFile(t *testing.T) { pubKeyBytes, _ := hex.DecodeString(pubKey) clientId := encryption.Hash(pubKeyBytes) - formFieldByt, err := json.Marshal(&allocation.UpdateFileChanger{BaseFileChanger: allocation.BaseFileChanger{Filename: `helper_integration_test.go`}}) + path := "/some_file" + formFieldByt, err := json.Marshal(&allocation.UpdateFileChanger{BaseFileChanger: allocation.BaseFileChanger{Filename: `helper_integration_test.go`, Path: path}}) if err != nil { t.Fatal(err) } @@ -68,7 +69,7 @@ func TestBlobberGRPCService_UploadFile(t *testing.T) { input: &blobbergrpc.UploadFileRequest{ Allocation: allocationTx, - Path: "/some_file", + Path: path, ConnectionId: "connection_id", Method: "POST", UploadMeta: string(formFieldByt), diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 80deec905..015d7de97 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -235,6 +235,55 @@ func GetReferenceFromLookupHash(ctx context.Context, allocationID, path_hash str return nil, err } +// GetRefType Select type from ref and return it +func GetRefType(ctx context.Context, allocationID, path string) (string, error) { + ref := new(Ref) + db := datastore.GetStore().GetTransaction(ctx) + err := db.Select("type").Where("allocation_id=? AND path=?", allocationID, path).First(ref).Error + if err != nil { + return "", err + } + return ref.Type, nil +} + +// GetRefWithID Return Ref with only ID selected in sql query +func GetRefWithID(ctx context.Context, allocationID, path string) (*Ref, error) { + ref := new(Ref) + db := datastore.GetStore().GetTransaction(ctx) + err := db.Select("id").Where("allocation_id=? AND path=?", allocationID, path).First(ref).Error + if err != nil { + return nil, err + } + return ref, nil +} + +// IsRefExist checks if ref with given path exists and returns error other than gorm.ErrRecordNotFound +func IsRefExist(ctx context.Context, allocationID, path string) (bool, error) { + db := datastore.GetStore().GetTransaction(ctx) + var count int64 + if err := db.Model(&Ref{}).Where("allocation_id=? AND path=?", allocationID, path).Count(&count).Error; err != nil { + return false, err + } + + return count > 0, nil +} + +// GetRefsTypeFromPaths Give list of paths it will return refs of respective path with only Type and Path selected in sql query +func GetRefsTypeFromPaths(ctx context.Context, allocationID string, paths []string) (refs []*Ref, err error) { + if len(paths) == 0 { + return + } + + db := datastore.GetStore().GetTransaction(ctx) + db = db.Select("path", "type") + for _, p := range paths { + db = db.Or(Ref{AllocationID: allocationID, Path: p}) + } + + err = db.Find(&refs).Error + return +} + func GetSubDirsFromPath(p string) []string { path := p parent, cur := filepath.Split(path) diff --git a/code/go/0chain.net/core/common/utils.go b/code/go/0chain.net/core/common/utils.go index 558bad211..9b68754de 100644 --- a/code/go/0chain.net/core/common/utils.go +++ b/code/go/0chain.net/core/common/utils.go @@ -1,6 +1,10 @@ package common -import "fmt" +import ( + "fmt" + "path/filepath" + "strings" +) // IsEmpty checks whether the input string is empty or not func IsEmpty(s string) bool { @@ -22,3 +26,23 @@ func ToKey(key interface{}) string { func IsEqual(key1, key2 string) bool { return key1 == key2 } + +// getParentPaths For path /a/b/c.txt, will return [/a,/a/b] +func GetParentPaths(fPath string) ([]string, error) { + if fPath == "" { + return nil, nil + } + + fPath = filepath.Clean(fPath) + if !filepath.IsAbs(fPath) { + return nil, NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fPath)) + } + splittedPaths := strings.Split(fPath, "/") + var paths []string + + for i := 0; i < len(splittedPaths); i++ { + subPath := strings.Join(splittedPaths[0:i], "/") + paths = append(paths, subPath) + } + return paths[2:], nil +} From b6e90ed9d42dd7e350511899fca72727e280b3bb Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 10 Mar 2022 21:13:07 +0800 Subject: [PATCH 123/161] fix(docker): fixed validator workdir issue (#581) --- docker.local/b0docker-compose-debug.yml | 10 +++++----- docker.local/b0docker-compose.yml | 10 +++++----- docker.local/docker-compose.yml | 10 +++++----- docker.local/https.docker-compose.yml | 10 +++++----- docker.local/p0docker-compose.yml | 10 +++++----- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docker.local/b0docker-compose-debug.yml b/docker.local/b0docker-compose-debug.yml index 8525d3500..de0a4c271 100644 --- a/docker.local/b0docker-compose-debug.yml +++ b/docker.local/b0docker-compose-debug.yml @@ -34,13 +34,13 @@ services: links: - postgres-post:postgres-post volumes: - - ../config:/blobber/config - - ./blobber${BLOBBER}/data:/blobber/data - - ./blobber${BLOBBER}/log:/blobber/log - - ./keys_config:/blobber/keysconfig + - ../config:/validator/config + - ./blobber${BLOBBER}/data:/validator/data + - ./blobber${BLOBBER}/log:/validator/log + - ./keys_config:/validator/keysconfig ports: - "506${BLOBBER}:506${BLOBBER}" - command: ./bin/validator --port 506${BLOBBER} --hostname 198.18.0.6${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --log_dir /blobber/log + command: ./bin/validator --port 506${BLOBBER} --hostname 198.18.0.6${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --log_dir /validator/log networks: default: testnet0: diff --git a/docker.local/b0docker-compose.yml b/docker.local/b0docker-compose.yml index f3e392acc..1b9ba8f4e 100644 --- a/docker.local/b0docker-compose.yml +++ b/docker.local/b0docker-compose.yml @@ -33,13 +33,13 @@ services: links: - postgres-post:postgres-post volumes: - - ../config:/blobber/config - - ./blobber${BLOBBER}/data:/blobber/data - - ./blobber${BLOBBER}/log:/blobber/log - - ./keys_config:/blobber/keysconfig + - ../config:/validator/config + - ./blobber${BLOBBER}/data:/validator/data + - ./blobber${BLOBBER}/log:/validator/log + - ./keys_config:/validator/keysconfig ports: - "506${BLOBBER}:506${BLOBBER}" - command: ./bin/validator --port 506${BLOBBER} --hostname 198.18.0.6${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --log_dir /blobber/log + command: ./bin/validator --port 506${BLOBBER} --hostname 198.18.0.6${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --log_dir /validator/log networks: default: testnet0: diff --git a/docker.local/docker-compose.yml b/docker.local/docker-compose.yml index 5d250bf16..bdafd0be6 100644 --- a/docker.local/docker-compose.yml +++ b/docker.local/docker-compose.yml @@ -33,15 +33,15 @@ services: links: - postgres-post:postgres-post volumes: - - ../config:/blobber/config - - ./blobber${BLOBBER}/data:/blobber/data - - ./blobber${BLOBBER}/log:/blobber/log - - ./keys_config:/blobber/keysconfig + - ../config:/validator/config + - ./blobber${BLOBBER}/data:/validator/data + - ./blobber${BLOBBER}/log:/validator/log + - ./keys_config:/validator/keysconfig ports: - "506${BLOBBER}:506${BLOBBER}" labels: zchain: "validator" - command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --log_dir /blobber/log + command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --log_dir /validator/log networks: default: testnet0: diff --git a/docker.local/https.docker-compose.yml b/docker.local/https.docker-compose.yml index 83e5b996b..45f776e75 100644 --- a/docker.local/https.docker-compose.yml +++ b/docker.local/https.docker-compose.yml @@ -33,15 +33,15 @@ services: links: - postgres-post:postgres-post volumes: - - ../config:/blobber/config - - ./blobber${BLOBBER}/data:/blobber/data - - ./blobber${BLOBBER}/log:/blobber/log - - ./keys_config:/blobber/keysconfig + - ../config:/validator/config + - ./blobber${BLOBBER}/data:/validator/data + - ./blobber${BLOBBER}/log:/validator/log + - ./keys_config:/validator/keysconfig ports: - "506${BLOBBER}:506${BLOBBER}" labels: zchain: "validator" - command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --log_dir /blobber/log + command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/bnode${BLOBBER}_keys.txt --log_dir /validator/log networks: default: testnet0: diff --git a/docker.local/p0docker-compose.yml b/docker.local/p0docker-compose.yml index 7c49f4bde..230711ed1 100644 --- a/docker.local/p0docker-compose.yml +++ b/docker.local/p0docker-compose.yml @@ -32,13 +32,13 @@ services: links: - postgres-post:postgres-post volumes: - - ../config:/blobber/config - - ./blobber${BLOBBER}/data:/blobber/data - - ./blobber${BLOBBER}/log:/blobber/log - - ./keys_config:/blobber/keysconfig + - ../config:/validator/config + - ./blobber${BLOBBER}/data:/validator/data + - ./blobber${BLOBBER}/log:/validator/log + - ./keys_config:/validator/keysconfig ports: - "506${BLOBBER}:506${BLOBBER}" - command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --log_dir /blobber/log + command: ./bin/validator --port 506${BLOBBER} --hostname localhost --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --log_dir /validator/log networks: default: testnet0: From c1c3e70a1b62f5d688b017b5a917ae0604bc465b Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 11 Mar 2022 14:05:21 +0800 Subject: [PATCH 124/161] fix(gomod): upgraded grpc (#580) --- go.mod | 7 ++++--- go.sum | 13 ++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index ae3b4c261..baa9bfa18 100644 --- a/go.mod +++ b/go.mod @@ -27,10 +27,11 @@ require ( go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.21.0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 - golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a - google.golang.org/grpc v1.44.0 + google.golang.org/genproto v0.0.0-20220308174144-ae0e22291548 + google.golang.org/grpc v1.45.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 google.golang.org/protobuf v1.27.1 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect diff --git a/go.sum b/go.sum index b4b537fa2..6e459c03c 100644 --- a/go.sum +++ b/go.sum @@ -1182,8 +1182,9 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1310,8 +1311,8 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 h1:8IVLkfbr2cLhv0a/vKq4UFUcJym8RmDoDboxCFWEjYE= +golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1525,8 +1526,9 @@ google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a h1:uqouglH745GoGeZ1YFZbPBiu961tgi/9Qm5jaorajjQ= google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220308174144-ae0e22291548 h1:J5ZNG1QIdstOl8aaUoFoQJfp04FKTsFV+jwkBHEchqs= +google.golang.org/genproto v0.0.0-20220308174144-ae0e22291548/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1561,8 +1563,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= From 3d825c8c3a1e5d1156e7cb69658c7ef0b13cfff2 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 12 Mar 2022 08:48:22 +0800 Subject: [PATCH 125/161] fix(resty): upgraded gosdk for restry (#576) --- code/go/0chain.net/core/transaction/http.go | 15 ++------------- go.mod | 2 +- go.sum | 6 ++---- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index dee99d909..d7adefdda 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -9,7 +9,6 @@ import ( "fmt" "io" - "net" "net/http" "net/url" @@ -93,13 +92,6 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] urls = append(urls, u+"?"+q.Encode()) } - transport := &http.Transport{ - Dial: (&net.Dialer{ - Timeout: resty.DefaultDialTimeout, - }).Dial, - TLSHandshakeTimeout: resty.DefaultDialTimeout, - } - header := map[string]string{ "Content-Type": "application/json; charset=utf-8", "Access-Control-Allow-Origin": "*", @@ -108,7 +100,7 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] //leave first item for ErrTooLessConfirmation var msgList = make([]string, 1, numSharders) - r := resty.New(transport, func(req *http.Request, resp *http.Response, respBody []byte, cancelFunc context.CancelFunc, err error) error { + r := resty.New(resty.WithHeader(header)).Then(func(req *http.Request, resp *http.Response, respBody []byte, cancelFunc context.CancelFunc, err error) error { if err != nil { //network issue msgList = append(msgList, err.Error()) return err @@ -143,10 +135,7 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] } return nil - }, - resty.WithTimeout(resty.DefaultRequestTimeout), - resty.WithRetry(resty.DefaultRetry), - resty.WithHeader(header)) + }) for { r.DoGet(context.TODO(), urls...) diff --git a/go.mod b/go.mod index baa9bfa18..def248e6f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da + github.com/0chain/gosdk v1.7.3-0.20220306071938-bb5b797cf02c github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index 6e459c03c..cd8cea3e5 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da h1:Mno3gifncwxsLEfkGUnsKqTvCDUbUioJXkYYH8Thw+8= -github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= +github.com/0chain/gosdk v1.7.3-0.20220306071938-bb5b797cf02c h1:sskqXnQdJlMvAt0E9fdW3bI545nFKtUlxsye4W7NoRo= +github.com/0chain/gosdk v1.7.3-0.20220306071938-bb5b797cf02c/go.mod h1:JF2dfUrSfIGK599LuUn7BgqQAj7oJPw+CE5GVSr/qEE= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -825,7 +825,6 @@ github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssy github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -878,7 +877,6 @@ github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= From 9a395363e734504d46983636a45d9960e52705f2 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 15 Mar 2022 14:25:32 +0800 Subject: [PATCH 126/161] fix(devops): enabled workflow_dispatch on dockerize job (#587) --- .github/workflows/build-&-publish-docker-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 333cf9287..ca21bb928 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -11,6 +11,7 @@ on: tags: - '*' pull_request: + workflow_dispatch: env: BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} From 4b23232bd21eb660973ca238b23a80644737cac5 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 16 Mar 2022 13:25:54 +0800 Subject: [PATCH 127/161] fix(copy):fixed panicking issue and missing stats bug (#584) --- .../allocation/allocationchange.go | 5 +- .../allocation/attributesfilechange.go | 4 +- .../blobbercore/allocation/copyfilechange.go | 48 +++++----- .../allocation/deletefilechange.go | 2 +- .../blobbercore/allocation/file_changer.go | 4 +- .../allocation/file_changer_add.go | 4 +- .../allocation/file_changer_update.go | 2 +- .../blobbercore/allocation/newfilechange.go | 2 +- .../allocation/renamefilechange.go | 2 +- .../allocation/renamefilechange_test.go | 2 +- .../allocation/updatefilechange_test.go | 2 +- .../blobbercore/handler/grpc_handler_test.go | 2 +- .../0chain.net/blobbercore/handler/handler.go | 2 +- .../handler/object_operation_handler.go | 32 +++---- .../blobbercore/writemarker/entity.go | 16 ++++ .../blobbercore/writemarker/protocol.go | 91 +++++++++---------- 16 files changed, 117 insertions(+), 103 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index aecd93a52..136814089 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -27,7 +27,7 @@ var OperationNotApplicable = common.NewError("operation_not_valid", "Not an appl type AllocationChangeProcessor interface { CommitToFileStore(ctx context.Context) error DeleteTempFile() error - ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) + ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) Marshal() (string, error) Unmarshal(string) error } @@ -142,10 +142,11 @@ func (cc *AllocationChangeCollector) ComputeProperties() { } } +// func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocationRoot string) error { for idx, change := range cc.Changes { changeProcessor := cc.AllocationChanges[idx] - _, err := changeProcessor.ProcessChange(ctx, change, allocationRoot) + _, err := changeProcessor.ApplyChange(ctx, change, allocationRoot) if err != nil { return err } diff --git a/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go b/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go index aeb57be64..dadc1220b 100644 --- a/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/attributesfilechange.go @@ -21,8 +21,8 @@ type AttributesChange struct { Attributes *reference.Attributes `json:"attributes"` // new attributes } -// ProcessChange processes the attributes changes. -func (ac *AttributesChange) ProcessChange(ctx context.Context, _ *AllocationChange, allocRoot string) (ref *reference.Ref, err error) { +// ApplyChange processes the attributes changes. +func (ac *AttributesChange) ApplyChange(ctx context.Context, _ *AllocationChange, allocRoot string) (ref *reference.Ref, err error) { var path, _ = filepath.Split(ac.Path) path = filepath.Clean(path) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 0851c5592..2763bba28 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -20,10 +20,10 @@ type CopyFileChange struct { } func (rf *CopyFileChange) DeleteTempFile() error { - return OperationNotApplicable + return nil } -func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { +func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { affectedRef, err := reference.GetObjectTree(ctx, rf.AllocationID, rf.SrcPath) if err != nil { return nil, err @@ -34,9 +34,17 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC if err != nil || destRef.Type != reference.DIRECTORY { return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.") } - rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot) + fileRefs := rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot) _, err = destRef.CalculateHash(ctx, true) + if err != nil { + return nil, err + } + + for _, fileRef := range fileRefs { + stats.NewFileCreated(ctx, fileRef.ID) + } + return destRef, err } @@ -92,7 +100,7 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC } dirRef.RemoveChild(childIndex) - rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot) + filerefs := rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot) dirRef.AddChild(destRef) _, err = rootRef.CalculateHash(ctx, true) @@ -100,29 +108,17 @@ func (rf *CopyFileChange) ProcessChange(ctx context.Context, change *AllocationC return nil, err } - err = rf.updateWriteMarker(ctx, destRef, affectedRef) + for _, fileRef := range filerefs { + stats.NewFileCreated(ctx, fileRef.ID) + } return rootRef, err } -func (rf *CopyFileChange) updateWriteMarker(ctx context.Context, destRef, affectedRef *reference.Ref) error { - ref := destRef - if affectedRef != nil { - for _, r := range destRef.Children { - if affectedRef.Name == r.Name { - ref = r - } - } - } - - if ref.Type == reference.FILE { - return stats.NewDirCreated(ctx, ref.ID) - } +func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, destRef *reference.Ref, allocationRoot string) []*reference.Ref { - return rf.updateWriteMarker(ctx, ref, nil) -} + var files []*reference.Ref -func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, destRef *reference.Ref, allocationRoot string) { if affectedRef.Type == reference.DIRECTORY { newRef := reference.NewDirectoryRef() newRef.AllocationID = rf.AllocationID @@ -132,10 +128,11 @@ func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, dest newRef.LookupHash = reference.GetReferenceLookup(newRef.AllocationID, newRef.Path) newRef.Attributes = datatypes.JSON(string(affectedRef.Attributes)) destRef.AddChild(newRef) + for _, childRef := range affectedRef.Children { - rf.processCopyRefs(ctx, childRef, newRef, allocationRoot) + files = append(files, rf.processCopyRefs(ctx, childRef, newRef, allocationRoot)...) } - } else { + } else if affectedRef.Type == reference.FILE { newFile := reference.NewFileRef() newFile.ActualFileHash = affectedRef.ActualFileHash newFile.ActualFileSize = affectedRef.ActualFileSize @@ -159,7 +156,12 @@ func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, dest newFile.ChunkSize = affectedRef.ChunkSize destRef.AddChild(newFile) + + files = append(files, newFile) } + + return files + } func (rf *CopyFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index c5896c367..bf50f314b 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -24,7 +24,7 @@ type DeleteFileChange struct { ContentHash map[string]bool } -func (nf *DeleteFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { +func (nf *DeleteFileChange) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { rootRef, contentHash, err := reference.DeleteObject(ctx, nf.AllocationID, nf.Path) if err != nil { return nil, err diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer.go b/code/go/0chain.net/blobbercore/allocation/file_changer.go index 705aef90b..04fe820ba 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer.go @@ -8,8 +8,8 @@ import ( // FileChanger file change processor type FileChanger interface { - // ProcessChange process change and save them on reference_objects - ProcessChange(ctx context.Context, + // ApplyChange process change and save them on reference_objects + ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) // Marshal marshal change as JSON string Marshal() (string, error) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_add.go b/code/go/0chain.net/blobbercore/allocation/file_changer_add.go index 434afa0d5..f272345f2 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_add.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_add.go @@ -18,8 +18,8 @@ type AddFileChanger struct { BaseFileChanger } -// ProcessChange update references, and create a new FileRef -func (nf *AddFileChanger) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { +// ApplyChange update references, and create a new FileRef +func (nf *AddFileChanger) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { path, _ := filepath.Split(nf.Path) path = filepath.Clean(path) tSubDirs := reference.GetSubDirsFromPath(path) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index c1568e48c..60ef44e61 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -22,7 +22,7 @@ type UpdateFileChanger struct { BaseFileChanger } -func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { +func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { path, _ := filepath.Split(nf.Path) path = filepath.Clean(path) tSubDirs := reference.GetSubDirsFromPath(path) diff --git a/code/go/0chain.net/blobbercore/allocation/newfilechange.go b/code/go/0chain.net/blobbercore/allocation/newfilechange.go index aa708caba..55689833d 100644 --- a/code/go/0chain.net/blobbercore/allocation/newfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/newfilechange.go @@ -118,7 +118,7 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a return rootRef, nil } -func (nf *NewFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { +func (nf *NewFileChange) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { if change.Operation == constants.FileOperationCreateDir { err := nf.Unmarshal(change.Input) if err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index 55e05a572..ed7dbb3e1 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -24,7 +24,7 @@ func (rf *RenameFileChange) DeleteTempFile() error { return OperationNotApplicable } -func (rf *RenameFileChange) ProcessChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { +func (rf *RenameFileChange) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) { isFilePresent, _ := reference.PathExists(ctx, rf.AllocationID, rf.NewName) if isFilePresent { return nil, common.NewError("invalid_reference_path", "file already exists") diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go index 9aafd38d8..08a8c279a 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -263,7 +263,7 @@ func TestBlobberCore_RenameFile(t *testing.T) { ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) change := &RenameFileChange{AllocationID: alloc.ID, Path: tc.path, NewName: tc.newName} - response, err := change.ProcessChange(ctx, tc.allocChange, tc.allocRoot) + response, err := change.ApplyChange(ctx, tc.allocChange, tc.allocRoot) if err != nil { if !tc.expectingError { diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index cf7164bf9..da334a2d9 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -284,7 +284,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { } _, err := func() (*reference.Ref, error) { - resp, err := change.ProcessChange(ctx, tc.allocChange, tc.allocRoot) + resp, err := change.ApplyChange(ctx, tc.allocChange, tc.allocRoot) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go index d2efa87b9..4e65a338b 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go @@ -59,7 +59,7 @@ func makeTestClient() (blobbergrpc.BlobberServiceClient, *grpc.ClientConn, error func makeTestAllocation(exp common.Timestamp) *allocation.Allocation { allocID := "allocation id" alloc := allocation.Allocation{ - Tx: "allocation tx", + Tx: "allocation id", ID: allocID, Terms: []*allocation.Terms{ { diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 33b2a19fb..cf4ae7f4a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -46,7 +46,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/file/upload/{allocation}", common.ToJSONResponse(WithConnection(UploadHandler))) r.HandleFunc("/v1/file/download/{allocation}", common.ToByteStream(WithConnection(DownloadHandler))).Methods(http.MethodGet, http.MethodOptions) r.HandleFunc("/v1/file/rename/{allocation}", common.ToJSONResponse(WithConnection(RenameHandler))).Methods(http.MethodPost, http.MethodOptions) - r.HandleFunc("/v1/file/copy/{allocation}", common.ToJSONResponse(WithConnection(CopyHandler))) + r.HandleFunc("/v1/file/copy/{allocation}", common.ToJSONResponse(WithConnection(CopyHandler))).Methods(http.MethodPost, http.MethodOptions) r.HandleFunc("/v1/file/attributes/{allocation}", common.ToJSONResponse(WithConnection(UpdateAttributesHandler))) r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods(http.MethodPost, http.MethodOptions) r.HandleFunc("/v1/dir/{allocation}", common.ToJSONResponse(WithConnection(CreateDirHandler))).Methods(http.MethodDelete, http.MethodOptions) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 85c0c6fcc..8885559c8 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -441,11 +441,11 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } var result blobberhttp.CommitResult - var latestWM *writemarker.WriteMarkerEntity + var latestWriteMarkerEntity *writemarker.WriteMarkerEntity if allocationObj.AllocationRoot == "" { - latestWM = nil + latestWriteMarkerEntity = nil } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, + latestWriteMarkerEntity, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) if err != nil { return nil, common.NewErrorf("latest_write_marker_read_error", @@ -453,16 +453,16 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } } - writemarkerObj := &writemarker.WriteMarkerEntity{} - writemarkerObj.WM = writeMarker + writemarkerEntity := &writemarker.WriteMarkerEntity{} + writemarkerEntity.WM = writeMarker - err = writemarkerObj.VerifyMarker(ctx, allocationObj, connectionObj) + err = writemarkerEntity.VerifyMarker(ctx, allocationObj, connectionObj) if err != nil { result.AllocationRoot = allocationObj.AllocationRoot result.ErrorMessage = "Verification of write marker failed: " + err.Error() result.Success = false - if latestWM != nil { - result.WriteMarker = &latestWM.WM + if latestWriteMarkerEntity != nil { + result.WriteMarker = &latestWriteMarkerEntity.WM } return &result, common.NewError("write_marker_verification_failed", result.ErrorMessage) } @@ -489,16 +489,16 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b if allocationRoot != writeMarker.AllocationRoot { result.AllocationRoot = allocationObj.AllocationRoot - if latestWM != nil { - result.WriteMarker = &latestWM.WM + if latestWriteMarkerEntity != nil { + result.WriteMarker = &latestWriteMarkerEntity.WM } result.Success = false result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root. Expected hash: " + allocationRoot return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage) } - writemarkerObj.ConnectionID = connectionObj.ConnectionID - writemarkerObj.ClientPublicKey = clientKey - err = writemarkerObj.Save(ctx) + writemarkerEntity.ConnectionID = connectionObj.ConnectionID + writemarkerEntity.ClientPublicKey = clientKey + err = writemarkerEntity.Save(ctx) if err != nil { return nil, common.NewError("write_marker_error", "Error persisting the write marker") } @@ -731,9 +731,7 @@ func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, r *http.R } func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (interface{}, error) { - if r.Method == "GET" { - return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") - } + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) if err != nil { @@ -781,7 +779,6 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int if err != nil { return nil, common.NewError("meta_error", "Error reading metadata for connection") } - mutex := lock.GetMutex(connectionObj.TableName(), connectionID) mutex.Lock() defer mutex.Unlock() @@ -791,7 +788,6 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } - newPath := filepath.Join(destPath, objectRef.Name) paths, err := common.GetParentPaths(newPath) if err != nil { diff --git a/code/go/0chain.net/blobbercore/writemarker/entity.go b/code/go/0chain.net/blobbercore/writemarker/entity.go index 586efef35..de0dc0acf 100644 --- a/code/go/0chain.net/blobbercore/writemarker/entity.go +++ b/code/go/0chain.net/blobbercore/writemarker/entity.go @@ -37,6 +37,7 @@ const ( ) type WriteMarkerEntity struct { + // WM new WriteMarker from client WM WriteMarker `gorm:"embedded"` Status WriteMarkerStatus `gorm:"column:status"` StatusMessage string `gorm:"column:status_message"` @@ -106,6 +107,21 @@ func GetWriteMarkerEntity(ctx context.Context, allocation_root string) (*WriteMa return wm, nil } +// AllocationRootMustUnique allocation_root must be unique in write_markers +func AllocationRootMustUnique(ctx context.Context, allocation_root string) error { + db := datastore.GetStore().GetTransaction(ctx) + + var c int64 + db.Raw("SELECT 1 FROM write_markers WHERE allocation_root = ? and status<>2 ", allocation_root). + Count(&c) + + if c > 0 { + return common.NewError("write_marker_validation_failed", "Duplicate write marker. Validation failed") + } + + return nil +} + func GetWriteMarkersInRange(ctx context.Context, allocationID, startAllocationRoot, endAllocationRoot string) ([]*WriteMarkerEntity, error) { db := datastore.GetStore().GetTransaction(ctx) var seqRange []int64 diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index 5571cf005..9d1e75d66 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -24,29 +24,24 @@ type CommitConnection struct { WriteMarker *WriteMarker `json:"write_marker"` } -func (wm *WriteMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.Allocation, co *allocation.AllocationChangeCollector) error { - if wm == nil { +// VerifyMarker verify WriteMarker's hash and check allocation_root if it is unique +func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *allocation.Allocation, co *allocation.AllocationChangeCollector) error { + if wme == nil { return common.NewError("invalid_write_marker", "No Write Marker was found") } - if wm.WM.PreviousAllocationRoot != sa.AllocationRoot { + if wme.WM.PreviousAllocationRoot != dbAllocation.AllocationRoot { return common.NewError("invalid_write_marker", "Invalid write marker. Prev Allocation root does not match the allocation root on record") } - if wm.WM.BlobberID != node.Self.ID { + if wme.WM.BlobberID != node.Self.ID { return common.NewError("write_marker_validation_failed", "Write Marker is not for the blobber") } - wmEntity, err := GetWriteMarkerEntity(ctx, wm.WM.AllocationRoot) - - if err == nil && wmEntity.Status != Failed { - return common.NewError("write_marker_validation_failed", "Duplicate write marker. Validation failed") - } - - if wm.WM.AllocationID != sa.ID { + if wme.WM.AllocationID != dbAllocation.ID { return common.NewError("write_marker_validation_failed", "Write Marker is not for the same allocation transaction") } - if wm.WM.Size != co.Size { - return common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker size %v does not match the connection size %v", wm.WM.Size, co.Size)) + if wme.WM.Size != co.Size { + return common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker size %v does not match the connection size %v", wme.WM.Size, co.Size)) } clientPublicKey := ctx.Value(constants.ContextKeyClientKey).(string) @@ -55,13 +50,13 @@ func (wm *WriteMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.Al } clientID := ctx.Value(constants.ContextKeyClient).(string) - if clientID == "" || clientID != wm.WM.ClientID || clientID != co.ClientID || co.ClientID != wm.WM.ClientID { + if clientID == "" || clientID != wme.WM.ClientID || clientID != co.ClientID || co.ClientID != wme.WM.ClientID { return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded") } - hashData := wm.WM.GetHashData() + hashData := wme.WM.GetHashData() signatureHash := encryption.Hash(hashData) - sigOK, err := encryption.Verify(clientPublicKey, wm.WM.Signature, signatureHash) + sigOK, err := encryption.Verify(clientPublicKey, wme.WM.Signature, signatureHash) if err != nil { return common.NewError("write_marker_validation_failed", "Error during verifying signature. "+err.Error()) } @@ -69,43 +64,47 @@ func (wm *WriteMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.Al return common.NewError("write_marker_validation_failed", "Write marker signature is not valid") } + if err := AllocationRootMustUnique(ctx, wme.WM.AllocationRoot); err != nil { + return err + } + return nil } -func (wm *WriteMarkerEntity) RedeemMarker(ctx context.Context) error { - if len(wm.CloseTxnID) > 0 { - t, err := transaction.VerifyTransaction(wm.CloseTxnID, chain.GetServerChain()) +func (wme *WriteMarkerEntity) RedeemMarker(ctx context.Context) error { + if len(wme.CloseTxnID) > 0 { + t, err := transaction.VerifyTransaction(wme.CloseTxnID, chain.GetServerChain()) if err == nil { - wm.Status = Committed - wm.StatusMessage = t.TransactionOutput - wm.CloseTxnID = t.Hash - err = wm.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) + wme.Status = Committed + wme.StatusMessage = t.TransactionOutput + wme.CloseTxnID = t.Hash + err = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) return err } } txn, err := transaction.NewTransactionEntity() if err != nil { - wm.StatusMessage = "Error creating transaction entity. " + err.Error() - wm.ReedeemRetries++ - if err := wm.UpdateStatus(ctx, Failed, "Error creating transaction entity. "+err.Error(), ""); err != nil { + wme.StatusMessage = "Error creating transaction entity. " + err.Error() + wme.ReedeemRetries++ + if err := wme.UpdateStatus(ctx, Failed, "Error creating transaction entity. "+err.Error(), ""); err != nil { Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) } return err } sn := &CommitConnection{} - sn.AllocationRoot = wm.WM.AllocationRoot - sn.PrevAllocationRoot = wm.WM.PreviousAllocationRoot - sn.WriteMarker = &wm.WM + sn.AllocationRoot = wme.WM.AllocationRoot + sn.PrevAllocationRoot = wme.WM.PreviousAllocationRoot + sn.WriteMarker = &wme.WM snBytes, err := json.Marshal(sn) if err != nil { Logger.Error("Error encoding sc input", zap.String("err:", err.Error()), zap.Any("scdata", sn)) - wm.Status = Failed - wm.StatusMessage = "Error encoding sc input. " + err.Error() - wm.ReedeemRetries++ - if err := wm.UpdateStatus(ctx, Failed, "Error encoding sc input. "+err.Error(), ""); err != nil { + wme.Status = Failed + wme.StatusMessage = "Error encoding sc input. " + err.Error() + wme.ReedeemRetries++ + if err := wme.UpdateStatus(ctx, Failed, "Error encoding sc input. "+err.Error(), ""); err != nil { Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) } return err @@ -114,10 +113,10 @@ func (wm *WriteMarkerEntity) RedeemMarker(ctx context.Context) error { err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.CLOSE_CONNECTION_SC_NAME, string(snBytes), 0) if err != nil { Logger.Error("Failed during sending close connection to the miner. ", zap.String("err:", err.Error())) - wm.Status = Failed - wm.StatusMessage = "Failed during sending close connection to the miner. " + err.Error() - wm.ReedeemRetries++ - if err := wm.UpdateStatus(ctx, Failed, "Failed during sending close connection to the miner. "+err.Error(), ""); err != nil { + wme.Status = Failed + wme.StatusMessage = "Failed during sending close connection to the miner. " + err.Error() + wme.ReedeemRetries++ + if err := wme.UpdateStatus(ctx, Failed, "Failed during sending close connection to the miner. "+err.Error(), ""); err != nil { Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) } return err @@ -127,18 +126,18 @@ func (wm *WriteMarkerEntity) RedeemMarker(ctx context.Context) error { t, err := transaction.VerifyTransaction(txn.Hash, chain.GetServerChain()) if err != nil { Logger.Error("Error verifying the close connection transaction", zap.String("err:", err.Error()), zap.String("txn", txn.Hash)) - wm.Status = Failed - wm.StatusMessage = "Error verifying the close connection transaction." + err.Error() - wm.ReedeemRetries++ - wm.CloseTxnID = txn.Hash - if err := wm.UpdateStatus(ctx, Failed, "Error verifying the close connection transaction."+err.Error(), txn.Hash); err != nil { + wme.Status = Failed + wme.StatusMessage = "Error verifying the close connection transaction." + err.Error() + wme.ReedeemRetries++ + wme.CloseTxnID = txn.Hash + if err := wme.UpdateStatus(ctx, Failed, "Error verifying the close connection transaction."+err.Error(), txn.Hash); err != nil { Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) } return err } - wm.Status = Committed - wm.StatusMessage = t.TransactionOutput - wm.CloseTxnID = t.Hash - err = wm.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) + wme.Status = Committed + wme.StatusMessage = t.TransactionOutput + wme.CloseTxnID = t.Hash + err = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) return err } From f54ed59e7850df38ec95b60809eab36309ec2220 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 17 Mar 2022 05:44:21 +0800 Subject: [PATCH 128/161] fix(gomod): upgraded testify (#593) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index def248e6f..372058a16 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/selvatico/go-mocket v1.0.7 github.com/spf13/afero v1.7.1 // indirect github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.21.0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 diff --git a/go.sum b/go.sum index cd8cea3e5..494d07979 100644 --- a/go.sum +++ b/go.sum @@ -954,8 +954,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= From ce1870a84580bada088de619e7cd75582059f3bc Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 17 Mar 2022 12:34:09 +0800 Subject: [PATCH 129/161] fix(init): try ping postgres to check if it is ready (#586) --- .../blobbercore/datastore/postgres.go | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index b622a3ba4..d511a1393 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -36,6 +36,10 @@ func (store *postgresStore) Open() error { return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) } + if err := sqldb.Ping(); err != nil { + return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) + } + sqldb.SetMaxIdleConns(100) sqldb.SetMaxOpenConns(200) sqldb.SetConnMaxLifetime(30 * time.Second) @@ -86,10 +90,8 @@ func (store *postgresStore) AutoMigrate() error { for i := 0; i < len(releases); i++ { v := releases[i] fmt.Print("\r + ", v.Version, " ") - isMigrated, err := store.IsMigrated(v) - if err != nil { - return err - } + isMigrated := store.IsMigrated(v) + if isMigrated { fmt.Print(" [SKIP]\n") continue @@ -108,20 +110,11 @@ func (store *postgresStore) AutoMigrate() error { return nil } -func (store *postgresStore) IsMigrated(m Migration) (bool, error) { - var version string - err := store.db. - Raw(`select version from "migrations" where version=?`, m.Version). - Pluck("version", &version). - Error +func (store *postgresStore) IsMigrated(m Migration) bool { - if err == nil { - return false, nil - } - - if err == gorm.ErrRecordNotFound { - return false, nil - } + var c int64 + store.db. + Raw(`SELECT 1 FROM "migrations" WHERE version=?`, m.Version).Count(&c) - return false, err + return c > 0 } From c6be906ad4b1e2d0ec0995995725bf54cfcfdb54 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 17 Mar 2022 13:59:20 +0800 Subject: [PATCH 130/161] fix(challenge):db conn is not closed (#591) --- .../blobbercore/challenge/challenge.go | 222 +++++++++--------- .../blobbercore/challenge/entity.go | 42 +++- .../blobbercore/filestore/mock_store.go | 2 +- 3 files changed, 144 insertions(+), 122 deletions(-) diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index 4a00b6f4f..c4288bbc8 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" @@ -41,58 +40,67 @@ func syncOpenChallenges(ctx context.Context) { if err != nil { logging.Logger.Error("[challenge]open: ", zap.Error(err)) - } else { - bytesReader := bytes.NewBuffer(retBytes) - - d := json.NewDecoder(bytesReader) - d.UseNumber() - errd := d.Decode(&blobberChallenges) - - if errd != nil { - logging.Logger.Error("[challenge]json: ", zap.Error(errd)) - } else { - for _, challengeObj := range blobberChallenges.Challenges { - if challengeObj == nil || challengeObj.ChallengeID == "" { - logging.Logger.Info("[challenge]open: No challenge entity from the challenge map") - continue - } - - tx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(tx) - _, err := GetChallengeEntity(tx, challengeObj.ChallengeID) - - // challenge is not synced in db yet - if errors.Is(err, gorm.ErrRecordNotFound) { - latestChallenge, err := GetLastChallengeEntity(tx) - - if err != nil { - if !errors.Is(err, gorm.ErrRecordNotFound) { - logging.Logger.Error("[challenge]db: ", zap.Error(err)) - continue - } - } - - isFirstChallengeInDatabase := challengeObj.PrevChallengeID == "" || latestChallenge == nil - isNextChallengeOnChain := latestChallenge == nil || latestChallenge.ChallengeID == challengeObj.PrevChallengeID - - if isFirstChallengeInDatabase || isNextChallengeOnChain { - logging.Logger.Info("[challenge]add: ", zap.String("challenge_id", challengeObj.ChallengeID)) - challengeObj.Status = Accepted - challengeObj.CreatedAt = common.ToTime(challengeObj.Created) - challengeObj.UpdatedAt = challengeObj.CreatedAt - - if err := challengeObj.Save(tx); err != nil { - logging.Logger.Error("[challenge]db: ", zap.String("challenge_id", challengeObj.ChallengeID), zap.Error(err)) - } - } else { - logging.Logger.Error("[challenge]Challenge chain is not valid") - } - } - db.Commit() - tx.Done() - } + return + } + + bytesReader := bytes.NewBuffer(retBytes) + d := json.NewDecoder(bytesReader) + d.UseNumber() + if err := d.Decode(&blobberChallenges); err != nil { + logging.Logger.Error("[challenge]json: ", zap.Error(err)) + return + } + + for _, challengeObj := range blobberChallenges.Challenges { + + if challengeObj == nil || challengeObj.ChallengeID == "" { + logging.Logger.Info("[challenge]open: No challenge entity from the challenge map") + continue } + + saveNewChallenge(challengeObj, ctx) } + +} + +func saveNewChallenge(nextChallenge *ChallengeEntity, ctx context.Context) { + defer func() { + if r := recover(); r != nil { + logging.Logger.Error("[recover]challenge", zap.Any("err", r)) + } + }() + + db := datastore.GetStore().GetDB() + if Exists(db, nextChallenge.ChallengeID) { + return + } + + lastChallengeID, err := getLastChallengeID(db) + + if err != nil { + logging.Logger.Error("[challenge]db: ", zap.Error(err)) + return + } + + isValid := nextChallenge.PrevChallengeID == "" || lastChallengeID == nextChallenge.PrevChallengeID + + // it is not First and Next challenge + if !isValid { + logging.Logger.Error("[challenge]Challenge chain is not valid") + return + } + + logging.Logger.Info("[challenge]add: ", zap.String("challenge_id", nextChallenge.ChallengeID)) + nextChallenge.Status = Accepted + nextChallenge.CreatedAt = common.ToTime(nextChallenge.Created) + nextChallenge.UpdatedAt = nextChallenge.CreatedAt + + if err := db.Transaction(func(tx *gorm.DB) error { + return nextChallenge.SaveWith(tx) + }); err != nil { + logging.Logger.Error("[challenge]db: ", zap.String("challenge_id", nextChallenge.ChallengeID), zap.Error(err)) + } + } // processAccepted read accepted challenge from db, and send them to validator to pass challenge @@ -102,8 +110,8 @@ func processAccepted(ctx context.Context) { logging.Logger.Error("[recover]challenge", zap.Any("err", r)) } }() - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) + + db := datastore.GetStore().GetDB() openchallenges := make([]*ChallengeEntity, 0) db.Where(ChallengeEntity{Status: Accepted}).Find(&openchallenges) if len(openchallenges) > 0 { @@ -116,43 +124,30 @@ func processAccepted(ctx context.Context) { continue } swg.Add() - go func(redeemCtx context.Context, challengeEntity *ChallengeEntity) { - defer swg.Done() - redeemCtx = datastore.GetStore().CreateTransaction(redeemCtx) - defer redeemCtx.Done() - err := loadValidationTickets(redeemCtx, challengeEntity) - if err != nil { - logging.Logger.Error("[challenge]validate: ", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) - return - } - db := datastore.GetStore().GetTransaction(redeemCtx) - err = db.Commit().Error - if err != nil { - logging.Logger.Error("[challenge]db: ", zap.Any("challenge_id", challengeEntity.ChallengeID), zap.Error(err)) - return - } - }(ctx, openchallenge) + go validateChallenge(&swg, openchallenge) } swg.Wait() } - db.Rollback() - rctx.Done() } -// loadValidationTickets load validation tickets for challenge -func loadValidationTickets(ctx context.Context, challengeObj *ChallengeEntity) error { - defer func() { - if r := recover(); r != nil { - logging.Logger.Error("[recover]LoadValidationTickets", zap.Any("err", r)) - } - }() +func validateChallenge(swg *sizedwaitgroup.SizedWaitGroup, challengeObj *ChallengeEntity) { + defer swg.Done() - err := challengeObj.LoadValidationTickets(ctx) - if err != nil { - logging.Logger.Error("[challenge]load: ", zap.String("challenge_id", challengeObj.ChallengeID), zap.Error(err)) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) + defer ctx.Done() + + db := datastore.GetStore().GetTransaction(ctx) + if err := challengeObj.LoadValidationTickets(ctx); err != nil { + logging.Logger.Error("[challenge]validate: ", zap.Any("challenge_id", challengeObj.ChallengeID), zap.Error(err)) + db.Rollback() + return } - return err + if err := db.Commit().Error; err != nil { + logging.Logger.Error("[challenge]db: ", zap.Any("challenge_id", challengeObj.ChallengeID), zap.Error(err)) + db.Rollback() + return + } } func commitProcessed(ctx context.Context) { @@ -162,40 +157,47 @@ func commitProcessed(ctx context.Context) { } }() - rctx := datastore.GetStore().CreateTransaction(ctx) - db := datastore.GetStore().GetTransaction(rctx) - openchallenges := make([]*ChallengeEntity, 0) + db := datastore.GetStore().GetDB() + var challenges []*ChallengeEntity db.Where(ChallengeEntity{Status: Processed}). Order("sequence"). - Find(&openchallenges) + Find(&challenges) - for _, openchallenge := range openchallenges { - logging.Logger.Info("Attempting to commit challenge", zap.Any("challenge_id", openchallenge.ChallengeID), zap.Any("openchallenge", openchallenge)) - if err := openchallenge.UnmarshalFields(); err != nil { - logging.Logger.Error("ChallengeEntity_UnmarshalFields", zap.String("challenge_id", openchallenge.ChallengeID), zap.Error(err)) - } + for _, challenge := range challenges { + commitChallenge(challenge) + } - redeemCtx := datastore.GetStore().CreateTransaction(ctx) - err := openchallenge.CommitChallenge(redeemCtx, false) - if err != nil { - logging.Logger.Error("Error committing to blockchain", - zap.Error(err), - zap.String("challenge_id", openchallenge.ChallengeID)) - } +} - db := datastore.GetStore().GetTransaction(redeemCtx) - db.Commit() - if err == nil && openchallenge.Status == Committed { - logging.Logger.Info("Challenge has been submitted to blockchain", - zap.Any("id", openchallenge.ChallengeID), - zap.String("txn", openchallenge.CommitTxnID)) - } else { - logging.Logger.Info("Challenge was not committed", zap.Any("challenge_id", openchallenge.ChallengeID)) - break - } +func commitChallenge(openchallenge *ChallengeEntity) { + logging.Logger.Info("Attempting to commit challenge", zap.Any("challenge_id", openchallenge.ChallengeID), zap.Any("openchallenge", openchallenge)) + if err := openchallenge.UnmarshalFields(); err != nil { + logging.Logger.Error("ChallengeEntity_UnmarshalFields", zap.String("challenge_id", openchallenge.ChallengeID), zap.Error(err)) + } + + ctx := datastore.GetStore().CreateTransaction(context.TODO()) + defer ctx.Done() + + db := datastore.GetStore().GetTransaction(ctx) + + if err := openchallenge.CommitChallenge(ctx, false); err != nil { + logging.Logger.Error("Error committing to blockchain", + zap.Error(err), + zap.String("challenge_id", openchallenge.ChallengeID)) + db.Rollback() + return } - db.Rollback() - rctx.Done() + if err := db.Commit(); err != nil { + logging.Logger.Info("Challenge was not committed", zap.Any("challenge_id", openchallenge.ChallengeID)) + db.Rollback() + return + } + + if openchallenge.Status == Committed { + logging.Logger.Info("Challenge has been submitted to blockchain", + zap.Any("id", openchallenge.ChallengeID), + zap.String("txn", openchallenge.CommitTxnID)) + } } diff --git a/code/go/0chain.net/blobbercore/challenge/entity.go b/code/go/0chain.net/blobbercore/challenge/entity.go index 94c5b0435..201854d39 100644 --- a/code/go/0chain.net/blobbercore/challenge/entity.go +++ b/code/go/0chain.net/blobbercore/challenge/entity.go @@ -2,7 +2,9 @@ package challenge import ( "context" + "database/sql" "encoding/json" + "errors" "fmt" "time" @@ -10,8 +12,10 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/gosdk/constants" "gorm.io/datatypes" + "gorm.io/gorm" ) type ( @@ -107,6 +111,11 @@ func unMarshalField(stringObj datatypes.JSON, dest interface{}) error { } func (cr *ChallengeEntity) Save(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + return cr.SaveWith(db) +} + +func (cr *ChallengeEntity) SaveWith(db *gorm.DB) error { err := marshalField(cr.Validators, &cr.ValidatorsString) if err != nil { return err @@ -124,7 +133,6 @@ func (cr *ChallengeEntity) Save(ctx context.Context) error { return err } - db := datastore.GetStore().GetTransaction(ctx) err = db.Save(cr).Error return err } @@ -173,16 +181,28 @@ func GetChallengeEntity(ctx context.Context, challengeID string) (*ChallengeEnti return cr, nil } -func GetLastChallengeEntity(ctx context.Context) (*ChallengeEntity, error) { - db := datastore.GetStore().GetTransaction(ctx) - cr := &ChallengeEntity{} - err := db.Order("sequence desc").First(cr).Error - if err != nil { - return nil, err +func getLastChallengeID(db *gorm.DB) (string, error) { + if db == nil { + return "", constants.ErrInvalidParameter } - err = cr.UnmarshalFields() - if err != nil { - return nil, err + + var challengeID string + + err := db.Raw("SELECT challenge_id FROM challenges ORDER BY sequence DESC LIMIT 1").Row().Scan(&challengeID) + + if err == nil || errors.Is(err, sql.ErrNoRows) { + return challengeID, nil } - return cr, nil + + return "", err +} + +// Exists check challenge if exists in db +func Exists(db *gorm.DB, challengeID string) bool { + + var count int64 + db.Raw("SELECT 1 FROM challenges WHERE challenge_id=?", challengeID).Count(&count) + + return count > 0 + } diff --git a/code/go/0chain.net/blobbercore/filestore/mock_store.go b/code/go/0chain.net/blobbercore/filestore/mock_store.go index 4ff6c312e..a5a9b36e9 100644 --- a/code/go/0chain.net/blobbercore/filestore/mock_store.go +++ b/code/go/0chain.net/blobbercore/filestore/mock_store.go @@ -112,7 +112,7 @@ func (ms *MockStore) addFileInDataObj(allocationID, contentHash string) { return } if ms.d == nil { - ms.d = make(map[string]map[string]bool, 0) + ms.d = make(map[string]map[string]bool) } dataObj := ms.d[allocationID] dataObj[contentHash] = true From 87e7ca589ed486770931be6d19703355786909b6 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 18 Mar 2022 04:46:04 +0800 Subject: [PATCH 131/161] fix(download):fixed Access-Control-Allow-Headers for wasm sdk (#595) --- .../blobbercore/handler/handler_middlewares.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go index 7b462589d..d5a29e18e 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go +++ b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go @@ -9,13 +9,6 @@ import ( ) func useCors(h http.Handler) http.Handler { - allowedHeaders := []string{ - "X-Requested-With", "Content-Type", - "X-App-Client-ID", "X-App-Client-Key", "X-App-Client-Signature", - "Access-Control-Allow-origin", "Access-Control-Request-Method", - } - - allowedOrigins := []string{"*"} allowedMethods := []string{"GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS"} @@ -27,8 +20,8 @@ func useCors(h http.Handler) http.Handler { } }() - w.Header().Add("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", ")) - w.Header().Add("Access-Control-Allow-Origin", strings.Join(allowedOrigins, ", ")) + w.Header().Add("Access-Control-Allow-Headers", "*") + w.Header().Add("Access-Control-Allow-Origin", "*") w.Header().Add("Access-Control-Allow-Methods", strings.Join(allowedMethods, ", ")) w.Header().Add("Access-Control-Allow-Credentials", "true") From d4cf005619f935a4589d38112f1e6d83abd6c850 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 23 Mar 2022 20:20:39 +0800 Subject: [PATCH 132/161] fix(gomod):upgraded gorm and grpc (#599) --- go.mod | 14 +++++++------- go.sum | 49 +++++++++++++++++++++++++------------------------ 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 372058a16..c6444ea3c 100644 --- a/go.mod +++ b/go.mod @@ -4,17 +4,18 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.7.3-0.20220306071938-bb5b797cf02c + github.com/0chain/gosdk v1.7.6 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-ini/ini v1.55.0 // indirect github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.8.0 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 github.com/herumi/bls-go-binary v1.0.1-0.20210830012634-a8e769d3b872 github.com/improbable-eng/grpc-web v0.15.0 github.com/jackc/pgx/v4 v4.15.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/mapstructure v1.4.3 @@ -26,19 +27,18 @@ require ( github.com/stretchr/testify v1.7.1 go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.21.0 - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 + golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect - golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - google.golang.org/genproto v0.0.0-20220308174144-ae0e22291548 + google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e google.golang.org/grpc v1.45.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 - google.golang.org/protobuf v1.27.1 + google.golang.org/protobuf v1.28.0 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.3.1 - gorm.io/gorm v1.23.2 + gorm.io/gorm v1.23.3 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 494d07979..d64e03b21 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.7.3-0.20220306071938-bb5b797cf02c h1:sskqXnQdJlMvAt0E9fdW3bI545nFKtUlxsye4W7NoRo= -github.com/0chain/gosdk v1.7.3-0.20220306071938-bb5b797cf02c/go.mod h1:JF2dfUrSfIGK599LuUn7BgqQAj7oJPw+CE5GVSr/qEE= +github.com/0chain/gosdk v1.7.6 h1:wuOmA846ZjcXD1HtPVGdvfWmz91hd9UQvQiEYknbD4k= +github.com/0chain/gosdk v1.7.6/go.mod h1:Cch/iwYKdBMnwcsFm2x9hG4CLYTVChCRFqA1TKRmRXU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -209,8 +209,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= -github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= @@ -255,8 +255,8 @@ github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPO github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/ethereum/go-ethereum v1.10.4/go.mod h1:nEE0TP5MtxGzOMd7egIrbPJMQBnhVU3ELNxhBglIzhg= -github.com/ethereum/go-ethereum v1.10.13 h1:DEYFP9zk+Gruf3ae1JOJVhNmxK28ee+sMELPLgYTXpA= -github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= +github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc= +github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -452,6 +452,7 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= @@ -460,8 +461,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.8.0 h1:/57CcG6YAO31FWwTVUnNl+2RcDUAaT55HtI71KQAXVw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.8.0/go.mod h1:/fckq3NE+vGiJsd4fDt4ge1XrK8cN+e5G5QWIzdg7Q8= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 h1:ESEyqQqXXFIcImj/BE8oKEX37Zsuceb2cZI+EL/zNCY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0/go.mod h1:XnLCLFp3tjoZJszVKjfpyAK6J8sYIcQXWQxmqLWF21I= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -610,16 +611,18 @@ github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -644,7 +647,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -1080,9 +1083,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1201,7 +1203,7 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1310,8 +1312,8 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 h1:8IVLkfbr2cLhv0a/vKq4UFUcJym8RmDoDboxCFWEjYE= -golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1525,9 +1527,8 @@ google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220308174144-ae0e22291548 h1:J5ZNG1QIdstOl8aaUoFoQJfp04FKTsFV+jwkBHEchqs= -google.golang.org/genproto v0.0.0-20220308174144-ae0e22291548/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg= +google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1562,7 +1563,6 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -1580,8 +1580,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1640,8 +1641,8 @@ gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.23.2 h1:xmq9QRMWL8HTJyhAUBXy8FqIIQCYESeKfJL4DoGKiWQ= -gorm.io/gorm v1.23.2/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.3 h1:jYh3nm7uLZkrMVfA8WVNjDZryKfr7W+HTlInVgKFJAg= +gorm.io/gorm v1.23.3/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 2562e810b077830d4a9deab9a0304708800dd708 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 24 Mar 2022 11:33:28 +0800 Subject: [PATCH 133/161] Feat/available after (#600) --- .../datastore/postgres_migration.go | 1 + .../datastore/postgres_releases.go | 18 +- .../0chain.net/blobbercore/handler/context.go | 7 +- .../0chain.net/blobbercore/handler/handler.go | 28 +- .../handler/handler_download_test.go | 2 +- .../handler/handler_middlewares.go | 1 - .../blobbercore/handler/handler_share_test.go | 558 ++++++++++++++++++ .../blobbercore/handler/handler_test.go | 381 +----------- .../handler/object_operation_handler.go | 5 + .../blobbercore/reference/shareinfo.go | 3 +- go.mod | 82 ++- go.sum | 38 +- sql/01-create-table.sql | 1 + 13 files changed, 685 insertions(+), 440 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/handler_share_test.go diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_migration.go b/code/go/0chain.net/blobbercore/datastore/postgres_migration.go index 153fd401c..d9f0f139d 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres_migration.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres_migration.go @@ -17,6 +17,7 @@ type Migration struct { // + increase index version if any index is changed // + increase column version if any column/constraint is changed Version string `gorm:"column:version;primary_key"` + Remark string `gorm:"column:remark"` CreatedAt time.Time `gorm:"column:created_at"` Scripts []string `gorm:"-"` } diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_releases.go b/code/go/0chain.net/blobbercore/datastore/postgres_releases.go index e6b83695d..08076c39f 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres_releases.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres_releases.go @@ -1,12 +1,16 @@ package datastore +import "time" + // releases migration histories var releases = []Migration{ - // { - // Version: "0.1.0", - // CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC), - // Scripts: []string{ - // "CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path,deleted_at);", - // }, - // }, + { + Version: "0.0.1", + Remark: "Added available_at on marketplace_share_info", + CreatedAt: time.Date(2022, 3, 21, 0, 0, 0, 0, time.UTC), + Scripts: []string{ + `ALTER TABLE marketplace_share_info + ADD COLUMN IF NOT EXISTS available_at timestamp without time zone NOT NULL DEFAULT now();`, + }, + }, } diff --git a/code/go/0chain.net/blobbercore/handler/context.go b/code/go/0chain.net/blobbercore/handler/context.go index 31cc9e561..856327bc8 100644 --- a/code/go/0chain.net/blobbercore/handler/context.go +++ b/code/go/0chain.net/blobbercore/handler/context.go @@ -82,15 +82,16 @@ type ErrorResponse struct { func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all. + w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") + w.Header().Set("Access-Control-Allow-Headers", "*") if r.Method == "OPTIONS" { - w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") - w.Header().Set("Access-Control-Allow-Headers", "*") + w.Header().Add("Access-Control-Max-Age", "3600") + w.WriteHeader(http.StatusNoContent) return } TryParseForm(r) - w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all. w.Header().Set("Content-Type", "application/json") ctx, err := WithAuth(r) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index cf4ae7f4a..b2f4e3de1 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "runtime/pprof" + "strconv" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -74,7 +75,8 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/getstats", common.ToJSONResponse(stats.GetStatsHandler)) //marketplace related - r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.ToJSONResponse(WithConnection(MarketPlaceShareInfoHandler))) + r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.ToJSONResponse(WithConnection(InsertShare))).Methods(http.MethodOptions, http.MethodPost) + r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.ToJSONResponse(WithConnection(RevokeShare))).Methods(http.MethodOptions, http.MethodDelete) // lightweight http handler without heavy postgres transaction to improve performance @@ -513,6 +515,7 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { encryptionPublicKey := r.FormValue("encryption_public_key") authTicketString := r.FormValue("auth_ticket") + availableAfter := r.FormValue("available_after") authTicket := &readmarker.AuthTicket{} err = json.Unmarshal([]byte(authTicketString), &authTicket) @@ -534,6 +537,16 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { return nil, err } + availableAt := common.Now() + + if len(availableAfter) > 0 { + a, err := strconv.ParseInt(availableAfter, 10, 64) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid available_after: "+err.Error()) + } + availableAt = common.Timestamp(a) + } + shareInfo := reference.ShareInfo{ OwnerID: authTicket.OwnerID, ClientID: authTicket.ClientID, @@ -541,6 +554,7 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { ReEncryptionKey: authTicket.ReEncryptionKey, ClientEncryptionPublicKey: encryptionPublicKey, ExpiryAt: common.ToTime(authTicket.Expiration), + AvailableAt: common.ToTime(availableAt), } existingShare, _ := reference.GetShareInfo(ctx, authTicket.ClientID, authTicket.FilePathHash) @@ -559,18 +573,6 @@ func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { return map[string]interface{}{"message": "Share info added successfully"}, nil } -func MarketPlaceShareInfoHandler(ctx context.Context, r *http.Request) (interface{}, error) { - if r.Method == "DELETE" { - return RevokeShare(ctx, r) - } - - if r.Method == "POST" { - return InsertShare(ctx, r) - } - - return nil, errors.New("invalid request method, only POST is allowed") -} - //PrintCSS - print the common css elements func PrintCSS(w http.ResponseWriter) { fmt.Fprintf(w, "