From 88246fd353256655640a159f29131cd8bcdf4662 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 10:03:54 -0500 Subject: [PATCH 01/19] Added ListOffChainData endpoint --- cmd/main.go | 4 +- db/db.go | 45 +++++++++++++++ db/db_test.go | 97 ++++++++++++++++++++++++++++++++ mocks/db.generated.go | 60 ++++++++++++++++++++ sequencer/tracker.go | 1 - services/datacom/datacom.go | 18 +++--- services/datacom/datacom_test.go | 2 +- services/sync/sync.go | 32 +++++++++-- services/sync/sync_test.go | 85 +++++++++++++++++++++++++++- 9 files changed, 323 insertions(+), 21 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index db51f51c..077c14a2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -142,11 +142,11 @@ func start(cliCtx *cli.Context) error { []rpc.Service{ { Name: sync.APISYNC, - Service: sync.NewSyncEndpoints(storage), + Service: sync.NewEndpoints(storage), }, { Name: datacom.APIDATACOM, - Service: datacom.NewDataComEndpoints( + Service: datacom.NewEndpoints( storage, pk, sequencerTracker, diff --git a/db/db.go b/db/db.go index cf392a9b..0232fb17 100644 --- a/db/db.go +++ b/db/db.go @@ -29,6 +29,7 @@ type DB interface { Exists(ctx context.Context, key common.Hash) bool GetOffChainData(ctx context.Context, key common.Hash, dbTx sqlx.QueryerContext) (types.ArgBytes, error) + ListOffChainData(ctx context.Context, keys []common.Hash, dbTx sqlx.QueryerContext) (map[common.Hash]types.ArgBytes, error) StoreOffChainData(ctx context.Context, od []types.OffChainData, dbTx sqlx.ExecerContext) error } @@ -218,6 +219,50 @@ func (db *pgDB) GetOffChainData(ctx context.Context, key common.Hash, dbTx sqlx. return common.FromHex(hexValue), nil } +// ListOffChainData returns values identified by the given keys +func (db *pgDB) ListOffChainData(ctx context.Context, keys []common.Hash, dbTx sqlx.QueryerContext) (map[common.Hash]types.ArgBytes, error) { + const listOffchainDataSQL = ` + SELECT key, value + FROM data_node.offchain_data + WHERE key IN (?); + ` + + preparedKeys := make([]string, len(keys)) + for i, key := range keys { + preparedKeys[i] = key.Hex() + } + + query, args, err := sqlx.In(listOffchainDataSQL, preparedKeys) + if err != nil { + return nil, err + } + + // sqlx.In returns queries with the `?` bindvar, we can rebind it for our backend + query = db.pg.Rebind(query) + + rows, err := db.querier(dbTx).QueryxContext(ctx, query, args...) + if err != nil { + return nil, err + } + + defer rows.Close() + + list := make(map[common.Hash]types.ArgBytes) + for rows.Next() { + data := struct { + Key string `db:"key"` + Value string `db:"value"` + }{} + if err = rows.StructScan(&data); err != nil { + return nil, err + } + + list[common.HexToHash(data.Key)] = common.FromHex(data.Value) + } + + return list, nil +} + func (db *pgDB) execer(dbTx sqlx.ExecerContext) sqlx.ExecerContext { if dbTx != nil { return dbTx diff --git a/db/db_test.go b/db/db_test.go index 8195b3aa..a0d13260 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -2,6 +2,7 @@ package db import ( "context" + "database/sql/driver" "errors" "testing" @@ -478,6 +479,102 @@ func Test_DB_GetOffChainData(t *testing.T) { } } +func Test_DB_ListOffChainData(t *testing.T) { + testTable := []struct { + name string + od []types.OffChainData + keys []common.Hash + expected map[common.Hash]types.ArgBytes + returnErr error + }{ + { + name: "successfully selected value", + od: []types.OffChainData{{ + Key: common.HexToHash("key1"), + Value: []byte("value1"), + }}, + keys: []common.Hash{ + common.BytesToHash([]byte("key1")), + }, + expected: map[common.Hash]types.ArgBytes{ + common.BytesToHash([]byte("key1")): []byte("value1"), + }, + }, + { + name: "error returned", + od: []types.OffChainData{{ + Key: common.HexToHash("key1"), + Value: []byte("value1"), + }}, + keys: []common.Hash{ + common.BytesToHash([]byte("key1")), + }, + returnErr: errors.New("test error"), + }, + { + name: "no rows", + od: []types.OffChainData{{ + Key: common.HexToHash("key1"), + Value: []byte("value1"), + }}, + keys: []common.Hash{ + common.BytesToHash([]byte("underfined")), + }, + returnErr: ErrStateNotSynchronized, + }, + } + + for _, tt := range testTable { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + db, mock, err := sqlmock.New() + require.NoError(t, err) + + defer db.Close() + + wdb := sqlx.NewDb(db, "postgres") + + // Seed data + seedOffchainData(t, wdb, mock, tt.od) + + preparedKeys := make([]driver.Value, len(tt.keys)) + for i, key := range tt.keys { + preparedKeys[i] = key.Hex() + } + + expected := mock.ExpectQuery(`SELECT key, value FROM data_node\.offchain_data WHERE key IN \(\$1\)`). + WithArgs(preparedKeys...) + + if tt.returnErr != nil { + expected = expected.WillReturnError(tt.returnErr) + } else { + returnData := sqlmock.NewRows([]string{"key", "value"}) + + for key, val := range tt.expected { + returnData = returnData.AddRow(key.Hex(), common.Bytes2Hex(val)) + } + + expected = expected.WillReturnRows(returnData) + } + + dbPG := New(wdb) + + data, err := dbPG.ListOffChainData(context.Background(), tt.keys, wdb) + if tt.returnErr != nil { + require.ErrorIs(t, err, tt.returnErr) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, data) + } + + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} + func Test_DB_Exist(t *testing.T) { testTable := []struct { name string diff --git a/mocks/db.generated.go b/mocks/db.generated.go index 8c6924fb..9c27e216 100644 --- a/mocks/db.generated.go +++ b/mocks/db.generated.go @@ -357,6 +357,66 @@ func (_c *DB_GetUnresolvedBatchKeys_Call) RunAndReturn(run func(context.Context) return _c } +// ListOffChainData provides a mock function with given fields: ctx, keys, dbTx +func (_m *DB) ListOffChainData(ctx context.Context, keys []common.Hash, dbTx sqlx.QueryerContext) (map[common.Hash]types.ArgBytes, error) { + ret := _m.Called(ctx, keys, dbTx) + + if len(ret) == 0 { + panic("no return value specified for ListOffChainData") + } + + var r0 map[common.Hash]types.ArgBytes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []common.Hash, sqlx.QueryerContext) (map[common.Hash]types.ArgBytes, error)); ok { + return rf(ctx, keys, dbTx) + } + if rf, ok := ret.Get(0).(func(context.Context, []common.Hash, sqlx.QueryerContext) map[common.Hash]types.ArgBytes); ok { + r0 = rf(ctx, keys, dbTx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[common.Hash]types.ArgBytes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []common.Hash, sqlx.QueryerContext) error); ok { + r1 = rf(ctx, keys, dbTx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DB_ListOffChainData_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListOffChainData' +type DB_ListOffChainData_Call struct { + *mock.Call +} + +// ListOffChainData is a helper method to define mock.On call +// - ctx context.Context +// - keys []common.Hash +// - dbTx sqlx.QueryerContext +func (_e *DB_Expecter) ListOffChainData(ctx interface{}, keys interface{}, dbTx interface{}) *DB_ListOffChainData_Call { + return &DB_ListOffChainData_Call{Call: _e.mock.On("ListOffChainData", ctx, keys, dbTx)} +} + +func (_c *DB_ListOffChainData_Call) Run(run func(ctx context.Context, keys []common.Hash, dbTx sqlx.QueryerContext)) *DB_ListOffChainData_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]common.Hash), args[2].(sqlx.QueryerContext)) + }) + return _c +} + +func (_c *DB_ListOffChainData_Call) Return(_a0 map[common.Hash]types.ArgBytes, _a1 error) *DB_ListOffChainData_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DB_ListOffChainData_Call) RunAndReturn(run func(context.Context, []common.Hash, sqlx.QueryerContext) (map[common.Hash]types.ArgBytes, error)) *DB_ListOffChainData_Call { + _c.Call.Return(run) + return _c +} + // StoreLastProcessedBlock provides a mock function with given fields: ctx, task, block, dbTx func (_m *DB) StoreLastProcessedBlock(ctx context.Context, task string, block uint64, dbTx sqlx.ExecerContext) error { ret := _m.Called(ctx, task, block, dbTx) diff --git a/sequencer/tracker.go b/sequencer/tracker.go index 9c9c78b5..ea79fbc8 100644 --- a/sequencer/tracker.go +++ b/sequencer/tracker.go @@ -70,7 +70,6 @@ func (st *Tracker) setAddr(addr common.Address) { // GetUrl returns the last known URL of the Sequencer func (st *Tracker) GetUrl() string { st.lock.Lock() - defer st.lock.Unlock() return st.url } diff --git a/services/datacom/datacom.go b/services/datacom/datacom.go index b61d8d4c..871e17a2 100644 --- a/services/datacom/datacom.go +++ b/services/datacom/datacom.go @@ -14,29 +14,27 @@ import ( // APIDATACOM is the namespace of the datacom service const APIDATACOM = "datacom" -// DataComEndpoints contains implementations for the "datacom" RPC endpoints -type DataComEndpoints struct { +// Endpoints contains implementations for the "datacom" RPC endpoints +type Endpoints struct { db db.DB txMan rpc.DBTxManager privateKey *ecdsa.PrivateKey sequencerTracker *sequencer.Tracker } -// NewDataComEndpoints returns DataComEndpoints -func NewDataComEndpoints( - db db.DB, privateKey *ecdsa.PrivateKey, sequencerTracker *sequencer.Tracker, -) *DataComEndpoints { - return &DataComEndpoints{ +// NewEndpoints returns Endpoints +func NewEndpoints(db db.DB, pk *ecdsa.PrivateKey, st *sequencer.Tracker) *Endpoints { + return &Endpoints{ db: db, - privateKey: privateKey, - sequencerTracker: sequencerTracker, + privateKey: pk, + sequencerTracker: st, } } // SignSequence generates the accumulated input hash aka accInputHash of the sequence and sign it. // After storing the data that will be sent hashed to the contract, it returns the signature. // This endpoint is only accessible to the sequencer -func (d *DataComEndpoints) SignSequence(signedSequence types.SignedSequence) (interface{}, rpc.Error) { +func (d *Endpoints) SignSequence(signedSequence types.SignedSequence) (interface{}, rpc.Error) { // Verify that the request comes from the sequencer sender, err := signedSequence.Signer() if err != nil { diff --git a/services/datacom/datacom_test.go b/services/datacom/datacom_test.go index 45c1a7ef..281b1076 100644 --- a/services/datacom/datacom_test.go +++ b/services/datacom/datacom_test.go @@ -89,7 +89,7 @@ func TestDataCom_SignSequence(t *testing.T) { signer = cfg.signer } - dce := NewDataComEndpoints(dbMock, signer, sequencer) + dce := NewEndpoints(dbMock, signer, sequencer) sig, err := dce.SignSequence(*signedSequence) if cfg.expectedError != "" { diff --git a/services/sync/sync.go b/services/sync/sync.go index 8bed2474..71b30a71 100644 --- a/services/sync/sync.go +++ b/services/sync/sync.go @@ -3,6 +3,8 @@ package sync import ( "context" + "github.com/ethereum/go-ethereum/common" + "github.com/0xPolygon/cdk-data-availability/db" "github.com/0xPolygon/cdk-data-availability/log" "github.com/0xPolygon/cdk-data-availability/rpc" @@ -12,21 +14,21 @@ import ( // APISYNC is the namespace of the sync service const APISYNC = "sync" -// SyncEndpoints contains implementations for the "zkevm" RPC endpoints -type SyncEndpoints struct { +// Endpoints contains implementations for the "zkevm" RPC endpoints +type Endpoints struct { db db.DB txMan rpc.DBTxManager } -// NewSyncEndpoints returns ZKEVMEndpoints -func NewSyncEndpoints(db db.DB) *SyncEndpoints { - return &SyncEndpoints{ +// NewEndpoints returns Endpoints +func NewEndpoints(db db.DB) *Endpoints { + return &Endpoints{ db: db, } } // GetOffChainData returns the image of the given hash -func (z *SyncEndpoints) GetOffChainData(hash types.ArgHash) (interface{}, rpc.Error) { +func (z *Endpoints) GetOffChainData(hash types.ArgHash) (interface{}, rpc.Error) { return z.txMan.NewDbTxScope(z.db, func(ctx context.Context, dbTx db.Tx) (interface{}, rpc.Error) { data, err := z.db.GetOffChainData(ctx, hash.Hash(), dbTx) if err != nil { @@ -37,3 +39,21 @@ func (z *SyncEndpoints) GetOffChainData(hash types.ArgHash) (interface{}, rpc.Er return data, nil }) } + +// ListOffChainData returns the list of images of the given hashes +func (z *Endpoints) ListOffChainData(hashes []types.ArgHash) (interface{}, rpc.Error) { + keys := make([]common.Hash, len(hashes)) + for i, hash := range hashes { + keys[i] = hash.Hash() + } + + return z.txMan.NewDbTxScope(z.db, func(ctx context.Context, dbTx db.Tx) (interface{}, rpc.Error) { + list, err := z.db.ListOffChainData(ctx, keys, dbTx) + if err != nil { + log.Errorf("failed to list the requested data from the DB: %v", err) + return nil, rpc.NewRPCError(rpc.DefaultErrorCode, "failed to list the requested data") + } + + return list, nil + }) +} diff --git a/services/sync/sync_test.go b/services/sync/sync_test.go index 2b928715..f45eafbe 100644 --- a/services/sync/sync_test.go +++ b/services/sync/sync_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" + "github.com/ethereum/go-ethereum/common" + "github.com/0xPolygon/cdk-data-availability/mocks" "github.com/0xPolygon/cdk-data-availability/types" "github.com/stretchr/testify/require" @@ -66,7 +68,7 @@ func TestSyncEndpoints_GetOffChainData(t *testing.T) { defer txMock.AssertExpectations(t) defer dbMock.AssertExpectations(t) - z := &SyncEndpoints{db: dbMock} + z := &Endpoints{db: dbMock} got, err := z.GetOffChainData(tt.hash) if tt.err != nil { @@ -79,3 +81,84 @@ func TestSyncEndpoints_GetOffChainData(t *testing.T) { }) } } + +func TestSyncEndpoints_ListOffChainData(t *testing.T) { + tests := []struct { + name string + hashes []types.ArgHash + data interface{} + dbErr error + txErr error + err error + }{ + { + name: "successfully got offchain data", + hashes: []types.ArgHash{}, + data: map[common.Hash]types.ArgBytes{ + common.Hash{}: types.ArgBytes("offchaindata"), + }, + }, + { + name: "db returns error", + hashes: []types.ArgHash{}, + data: map[common.Hash]types.ArgBytes{ + common.Hash{}: types.ArgBytes("offchaindata"), + }, + dbErr: errors.New("test error"), + err: errors.New("failed to list the requested data"), + }, + { + name: "tx returns error", + hashes: []types.ArgHash{}, + data: map[common.Hash]types.ArgBytes{ + common.Hash{}: types.ArgBytes("offchaindata"), + }, + txErr: errors.New("test error"), + err: errors.New("failed to connect to the state"), + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + txMock := mocks.NewTx(t) + + dbMock := mocks.NewDB(t) + dbMock.On("BeginStateTransaction", context.Background()). + Return(txMock, tt.txErr) + if tt.txErr == nil { + keys := make([]common.Hash, len(tt.hashes)) + for i, hash := range tt.hashes { + keys[i] = hash.Hash() + } + + dbMock.On("ListOffChainData", context.Background(), keys, txMock). + Return(tt.data, tt.dbErr) + + if tt.err != nil { + txMock.On("Rollback"). + Return(nil) + } else { + txMock.On("Commit"). + Return(nil) + } + } + + defer txMock.AssertExpectations(t) + defer dbMock.AssertExpectations(t) + + z := &Endpoints{db: dbMock} + + got, err := z.ListOffChainData(tt.hashes) + if tt.err != nil { + require.Error(t, err) + require.EqualError(t, tt.err, err.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tt.data, got) + } + }) + } +} From 95ab82d29dcdcde1c1fde86141d4dcc9fa4bfeb6 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 11:01:22 -0500 Subject: [PATCH 02/19] Added ListOffChainData endpoint to client --- client/client.go | 25 ++++++++++- client/client_test.go | 82 +++++++++++++++++++++++++++++++++- db/db_test.go | 4 +- mocks/client.generated.go | 73 +++++++++++++++++++++++++++--- services/sync/sync.go | 3 +- services/sync/sync_test.go | 3 +- test/e2e/datacommittee_test.go | 24 +++++++--- 7 files changed, 190 insertions(+), 24 deletions(-) diff --git a/client/client.go b/client/client.go index fe26b194..a6632158 100644 --- a/client/client.go +++ b/client/client.go @@ -17,7 +17,8 @@ type Factory interface { // Client is the interface that defines the implementation of all the endpoints type Client interface { - GetOffChainData(ctx context.Context, hash common.Hash) ([]byte, error) + GetOffChainData(ctx context.Context, hash common.Hash) (types.ArgBytes, error) + ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash]types.ArgBytes, error) SignSequence(signedSequence types.SignedSequence) ([]byte, error) } @@ -67,7 +68,7 @@ func (c *client) SignSequence(signedSequence types.SignedSequence) ([]byte, erro } // GetOffChainData returns data based on it's hash -func (c *client) GetOffChainData(ctx context.Context, hash common.Hash) ([]byte, error) { +func (c *client) GetOffChainData(ctx context.Context, hash common.Hash) (types.ArgBytes, error) { response, err := rpc.JSONRPCCallWithContext(ctx, c.url, "sync_getOffChainData", hash) if err != nil { return nil, err @@ -84,3 +85,23 @@ func (c *client) GetOffChainData(ctx context.Context, hash common.Hash) ([]byte, return result, nil } + +// ListOffChainData returns data based on the given hashes +func (c *client) ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash]types.ArgBytes, error) { + response, err := rpc.JSONRPCCallWithContext(ctx, c.url, "sync_listOffChainData", hashes) + if err != nil { + return nil, err + } + + if response.Error != nil { + return nil, fmt.Errorf("%v %v", response.Error.Code, response.Error.Message) + } + + result := make(map[common.Hash]types.ArgBytes) + fmt.Println(string(response.Result)) + if err = json.Unmarshal(response.Result, &result); err != nil { + return nil, err + } + + return result, nil +} diff --git a/client/client_test.go b/client/client_test.go index 2eb73530..7c5814ba 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -104,7 +104,7 @@ func TestClient_GetOffChainData(t *testing.T) { name string hash common.Hash result string - data []byte + data types.ArgBytes statusCode int err error }{ @@ -112,7 +112,7 @@ func TestClient_GetOffChainData(t *testing.T) { name: "successfully got offhcain data", hash: common.BytesToHash([]byte("hash")), result: fmt.Sprintf(`{"result":"%s"}`, hex.EncodeToString([]byte("offchaindata"))), - data: []byte("offchaindata"), + data: types.ArgBytes("offchaindata"), }, { name: "error returned by server", @@ -169,3 +169,81 @@ func TestClient_GetOffChainData(t *testing.T) { }) } } + +func TestClient_ListOffChainData(t *testing.T) { + tests := []struct { + name string + hashes []common.Hash + result string + data map[common.Hash]types.ArgBytes + statusCode int + err error + }{ + { + name: "successfully got offhcain data", + hashes: []common.Hash{common.BytesToHash([]byte("hash"))}, + result: fmt.Sprintf(`{"result":{"%s":"%s"}}`, + common.BytesToHash([]byte("hash")).Hex(), hex.EncodeToString([]byte("offchaindata"))), + data: map[common.Hash]types.ArgBytes{ + common.BytesToHash([]byte("hash")): []byte("offchaindata"), + }, + }, + { + name: "error returned by server", + hashes: []common.Hash{common.BytesToHash([]byte("hash"))}, + result: `{"error":{"code":123,"message":"test error"}}`, + err: errors.New("123 test error"), + }, + { + name: "invalid offchain data returned by server", + hashes: []common.Hash{common.BytesToHash([]byte("hash"))}, + result: fmt.Sprintf(`{"result":{"%s":"invalid-signature"}}`, + common.BytesToHash([]byte("hash")).Hex()), + data: map[common.Hash]types.ArgBytes{ + common.BytesToHash([]byte("hash")): nil, + }, + }, + { + name: "unsuccessful status code returned by server", + hashes: []common.Hash{common.BytesToHash([]byte("hash"))}, + statusCode: http.StatusUnauthorized, + err: errors.New("invalid status code, expected: 200, found: 401"), + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var res rpc.Request + require.NoError(t, json.NewDecoder(r.Body).Decode(&res)) + require.Equal(t, "sync_listOffChainData", res.Method) + + var params [][]common.Hash + require.NoError(t, json.Unmarshal(res.Params, ¶ms)) + require.Equal(t, tt.hashes, params[0]) + + if tt.statusCode > 0 { + w.WriteHeader(tt.statusCode) + } + + _, err := fmt.Fprint(w, tt.result) + require.NoError(t, err) + })) + defer svr.Close() + + c := &client{url: svr.URL} + + got, err := c.ListOffChainData(context.Background(), tt.hashes) + if tt.err != nil { + require.Error(t, err) + require.EqualError(t, tt.err, err.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tt.data, got) + } + }) + } +} diff --git a/db/db_test.go b/db/db_test.go index a0d13260..f5c9e921 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -549,7 +549,7 @@ func Test_DB_ListOffChainData(t *testing.T) { WithArgs(preparedKeys...) if tt.returnErr != nil { - expected = expected.WillReturnError(tt.returnErr) + expected.WillReturnError(tt.returnErr) } else { returnData := sqlmock.NewRows([]string{"key", "value"}) @@ -557,7 +557,7 @@ func Test_DB_ListOffChainData(t *testing.T) { returnData = returnData.AddRow(key.Hex(), common.Bytes2Hex(val)) } - expected = expected.WillReturnRows(returnData) + expected.WillReturnRows(returnData) } dbPG := New(wdb) diff --git a/mocks/client.generated.go b/mocks/client.generated.go index 1bcf5872..4fc7d8c5 100644 --- a/mocks/client.generated.go +++ b/mocks/client.generated.go @@ -26,23 +26,23 @@ func (_m *Client) EXPECT() *Client_Expecter { } // GetOffChainData provides a mock function with given fields: ctx, hash -func (_m *Client) GetOffChainData(ctx context.Context, hash common.Hash) ([]byte, error) { +func (_m *Client) GetOffChainData(ctx context.Context, hash common.Hash) (types.ArgBytes, error) { ret := _m.Called(ctx, hash) if len(ret) == 0 { panic("no return value specified for GetOffChainData") } - var r0 []byte + var r0 types.ArgBytes var r1 error - if rf, ok := ret.Get(0).(func(context.Context, common.Hash) ([]byte, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, common.Hash) (types.ArgBytes, error)); ok { return rf(ctx, hash) } - if rf, ok := ret.Get(0).(func(context.Context, common.Hash) []byte); ok { + if rf, ok := ret.Get(0).(func(context.Context, common.Hash) types.ArgBytes); ok { r0 = rf(ctx, hash) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) + r0 = ret.Get(0).(types.ArgBytes) } } @@ -74,12 +74,71 @@ func (_c *Client_GetOffChainData_Call) Run(run func(ctx context.Context, hash co return _c } -func (_c *Client_GetOffChainData_Call) Return(_a0 []byte, _a1 error) *Client_GetOffChainData_Call { +func (_c *Client_GetOffChainData_Call) Return(_a0 types.ArgBytes, _a1 error) *Client_GetOffChainData_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Client_GetOffChainData_Call) RunAndReturn(run func(context.Context, common.Hash) (types.ArgBytes, error)) *Client_GetOffChainData_Call { + _c.Call.Return(run) + return _c +} + +// ListOffChainData provides a mock function with given fields: ctx, hashes +func (_m *Client) ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash]types.ArgBytes, error) { + ret := _m.Called(ctx, hashes) + + if len(ret) == 0 { + panic("no return value specified for ListOffChainData") + } + + var r0 map[common.Hash]types.ArgBytes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []common.Hash) (map[common.Hash]types.ArgBytes, error)); ok { + return rf(ctx, hashes) + } + if rf, ok := ret.Get(0).(func(context.Context, []common.Hash) map[common.Hash]types.ArgBytes); ok { + r0 = rf(ctx, hashes) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[common.Hash]types.ArgBytes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []common.Hash) error); ok { + r1 = rf(ctx, hashes) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Client_ListOffChainData_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListOffChainData' +type Client_ListOffChainData_Call struct { + *mock.Call +} + +// ListOffChainData is a helper method to define mock.On call +// - ctx context.Context +// - hashes []common.Hash +func (_e *Client_Expecter) ListOffChainData(ctx interface{}, hashes interface{}) *Client_ListOffChainData_Call { + return &Client_ListOffChainData_Call{Call: _e.mock.On("ListOffChainData", ctx, hashes)} +} + +func (_c *Client_ListOffChainData_Call) Run(run func(ctx context.Context, hashes []common.Hash)) *Client_ListOffChainData_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]common.Hash)) + }) + return _c +} + +func (_c *Client_ListOffChainData_Call) Return(_a0 map[common.Hash]types.ArgBytes, _a1 error) *Client_ListOffChainData_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Client_GetOffChainData_Call) RunAndReturn(run func(context.Context, common.Hash) ([]byte, error)) *Client_GetOffChainData_Call { +func (_c *Client_ListOffChainData_Call) RunAndReturn(run func(context.Context, []common.Hash) (map[common.Hash]types.ArgBytes, error)) *Client_ListOffChainData_Call { _c.Call.Return(run) return _c } diff --git a/services/sync/sync.go b/services/sync/sync.go index 71b30a71..f9741480 100644 --- a/services/sync/sync.go +++ b/services/sync/sync.go @@ -3,12 +3,11 @@ package sync import ( "context" - "github.com/ethereum/go-ethereum/common" - "github.com/0xPolygon/cdk-data-availability/db" "github.com/0xPolygon/cdk-data-availability/log" "github.com/0xPolygon/cdk-data-availability/rpc" "github.com/0xPolygon/cdk-data-availability/types" + "github.com/ethereum/go-ethereum/common" ) // APISYNC is the namespace of the sync service diff --git a/services/sync/sync_test.go b/services/sync/sync_test.go index f45eafbe..9101c4fd 100644 --- a/services/sync/sync_test.go +++ b/services/sync/sync_test.go @@ -5,10 +5,9 @@ import ( "errors" "testing" - "github.com/ethereum/go-ethereum/common" - "github.com/0xPolygon/cdk-data-availability/mocks" "github.com/0xPolygon/cdk-data-availability/types" + "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" ) diff --git a/test/e2e/datacommittee_test.go b/test/e2e/datacommittee_test.go index f6984c97..23d48042 100644 --- a/test/e2e/datacommittee_test.go +++ b/test/e2e/datacommittee_test.go @@ -204,11 +204,13 @@ func TestDataCommittee(t *testing.T) { expectedKeys, err := getSequenceBatchesKeys(clientL1, iter.Event) require.NoError(t, err) for _, m := range membs { + offchainData, err := listOffchainDataKeys(m, expectedKeys) + require.NoError(t, err) + require.Len(t, offchainData, len(expectedKeys)) + // Each member (including m0) should have all the keys for _, expected := range expectedKeys { - actual, err := getOffchainDataKeys(m, expected) - require.NoError(t, err) - require.Equal(t, expected, actual) + require.Equal(t, expected, offchainData[expected]) } } } @@ -239,16 +241,24 @@ func getSequenceBatchesKeys(clientL1 *ethclient.Client, event *polygonvalidium.P return keys, err } -func getOffchainDataKeys(m member, tx common.Hash) (common.Hash, error) { +func listOffchainDataKeys(m member, txes []common.Hash) (map[common.Hash]common.Hash, error) { testUrl := fmt.Sprintf("http://127.0.0.1:420%d", m.i) mc := newTestClient(testUrl, m.addr) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - data, err := mc.client.GetOffChainData(ctx, tx) + + data, err := mc.client.ListOffChainData(ctx, txes) if err != nil { - return common.Hash{}, err + return nil, err } - return crypto.Keccak256Hash(data), nil + + preparedData := make(map[common.Hash]common.Hash) + for hash, val := range data { + preparedData[hash] = crypto.Keccak256Hash(val) + } + + return preparedData, nil } type member struct { From 3fa57fae77d5eb9aafe5f761ca029a4f79ab34a1 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 11:05:41 -0500 Subject: [PATCH 03/19] Added ListOffChainData endpoint to client --- services/sync/sync_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/sync/sync_test.go b/services/sync/sync_test.go index 9101c4fd..a207fd4f 100644 --- a/services/sync/sync_test.go +++ b/services/sync/sync_test.go @@ -94,14 +94,14 @@ func TestSyncEndpoints_ListOffChainData(t *testing.T) { name: "successfully got offchain data", hashes: []types.ArgHash{}, data: map[common.Hash]types.ArgBytes{ - common.Hash{}: types.ArgBytes("offchaindata"), + common.BytesToHash(nil): types.ArgBytes("offchaindata"), }, }, { name: "db returns error", hashes: []types.ArgHash{}, data: map[common.Hash]types.ArgBytes{ - common.Hash{}: types.ArgBytes("offchaindata"), + common.BytesToHash(nil): types.ArgBytes("offchaindata"), }, dbErr: errors.New("test error"), err: errors.New("failed to list the requested data"), @@ -110,7 +110,7 @@ func TestSyncEndpoints_ListOffChainData(t *testing.T) { name: "tx returns error", hashes: []types.ArgHash{}, data: map[common.Hash]types.ArgBytes{ - common.Hash{}: types.ArgBytes("offchaindata"), + common.BytesToHash(nil): types.ArgBytes("offchaindata"), }, txErr: errors.New("test error"), err: errors.New("failed to connect to the state"), From bf52c53de043fc73b89d602664a2ed8352962d47 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 11:11:53 -0500 Subject: [PATCH 04/19] Updated e2e tests --- test/e2e/e2e_test.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e36a6010..e9b6733e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -116,15 +116,20 @@ func (tc *testClient) signSequence(t *testing.T, expected *types.SignedSequence, actualAddr, err := expected.Signer() require.NoError(t, err) assert.Equal(t, tc.dacMemberAddr, actualAddr) + // Check that offchain data has been stored expectedOffchainData := expected.Sequence.OffChainData() + hashes := make([]common.Hash, len(expectedOffchainData)) + for i, od := range expectedOffchainData { + hashes[i] = od.Key + } + + actualData, err := tc.client.ListOffChainData(context.Background(), hashes) + require.NoError(t, err) + assert.Len(t, actualData, len(expectedOffchainData)) + for _, od := range expectedOffchainData { - actualData, err := tc.client.GetOffChainData( - context.Background(), - od.Key, - ) - require.NoError(t, err) - assert.Equal(t, od.Value, actualData) + assert.Equal(t, od.Value, actualData[od.Key]) } } } From 5bcc16a2a707b6c6ebc8e6d22b745a5afc33acad Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 12:14:39 -0500 Subject: [PATCH 05/19] Added debug logs --- client/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index a6632158..099fe859 100644 --- a/client/client.go +++ b/client/client.go @@ -98,10 +98,12 @@ func (c *client) ListOffChainData(ctx context.Context, hashes []common.Hash) (ma } result := make(map[common.Hash]types.ArgBytes) - fmt.Println(string(response.Result)) if err = json.Unmarshal(response.Result, &result); err != nil { return nil, err } + fmt.Println("result:", string(response.Result)) + fmt.Printf("result: %v\n", result) + return result, nil } From fc69b255c2615fd6443a1b6b7a740214fb8b7fb0 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 12:23:02 -0500 Subject: [PATCH 06/19] Added debug logs --- client/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/client.go b/client/client.go index 099fe859..1f0800e2 100644 --- a/client/client.go +++ b/client/client.go @@ -102,6 +102,7 @@ func (c *client) ListOffChainData(ctx context.Context, hashes []common.Hash) (ma return nil, err } + fmt.Println("hashes:", hashes) fmt.Println("result:", string(response.Result)) fmt.Printf("result: %v\n", result) From 481f84752d16bbbad39bf8dc206a9ff9565019c4 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 12:40:07 -0500 Subject: [PATCH 07/19] Added debug logs --- services/sync/sync.go | 2 +- test/e2e/datacommittee_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/services/sync/sync.go b/services/sync/sync.go index f9741480..11eae652 100644 --- a/services/sync/sync.go +++ b/services/sync/sync.go @@ -50,7 +50,7 @@ func (z *Endpoints) ListOffChainData(hashes []types.ArgHash) (interface{}, rpc.E list, err := z.db.ListOffChainData(ctx, keys, dbTx) if err != nil { log.Errorf("failed to list the requested data from the DB: %v", err) - return nil, rpc.NewRPCError(rpc.DefaultErrorCode, "failed to list the requested data") + return "0x0", rpc.NewRPCError(rpc.DefaultErrorCode, "failed to list the requested data") } return list, nil diff --git a/test/e2e/datacommittee_test.go b/test/e2e/datacommittee_test.go index 23d48042..8742d672 100644 --- a/test/e2e/datacommittee_test.go +++ b/test/e2e/datacommittee_test.go @@ -204,6 +204,15 @@ func TestDataCommittee(t *testing.T) { expectedKeys, err := getSequenceBatchesKeys(clientL1, iter.Event) require.NoError(t, err) for _, m := range membs { + // Each member (including m0) should have all the keys + for _, expected := range expectedKeys { + actual, err := getOffchainDataKeys(m, expected) + require.NoError(t, err) + require.Equal(t, expected, actual) + } + + continue + offchainData, err := listOffchainDataKeys(m, expectedKeys) require.NoError(t, err) require.Len(t, offchainData, len(expectedKeys)) @@ -241,6 +250,18 @@ func getSequenceBatchesKeys(clientL1 *ethclient.Client, event *polygonvalidium.P return keys, err } +func getOffchainDataKeys(m member, tx common.Hash) (common.Hash, error) { + testUrl := fmt.Sprintf("http://127.0.0.1:420%d", m.i) + mc := newTestClient(testUrl, m.addr) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + data, err := mc.client.GetOffChainData(ctx, tx) + if err != nil { + return common.Hash{}, err + } + return crypto.Keccak256Hash(data), nil +} + func listOffchainDataKeys(m member, txes []common.Hash) (map[common.Hash]common.Hash, error) { testUrl := fmt.Sprintf("http://127.0.0.1:420%d", m.i) mc := newTestClient(testUrl, m.addr) From ab0bde770ec76c19bfed26601a5b0dd726fa9e8a Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 12:47:51 -0500 Subject: [PATCH 08/19] Fixed nit --- sequencer/tracker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sequencer/tracker.go b/sequencer/tracker.go index ea79fbc8..9c9c78b5 100644 --- a/sequencer/tracker.go +++ b/sequencer/tracker.go @@ -70,6 +70,7 @@ func (st *Tracker) setAddr(addr common.Address) { // GetUrl returns the last known URL of the Sequencer func (st *Tracker) GetUrl() string { st.lock.Lock() + defer st.lock.Unlock() return st.url } From 14271d450c9b36d93fdc71057c51fb1313e5c23e Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 12:57:17 -0500 Subject: [PATCH 09/19] Fixed nit --- client/client.go | 19 ++++++++++++------- mocks/client.generated.go | 28 ++++++++++++++-------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/client/client.go b/client/client.go index 1f0800e2..a1f1bfa8 100644 --- a/client/client.go +++ b/client/client.go @@ -17,8 +17,8 @@ type Factory interface { // Client is the interface that defines the implementation of all the endpoints type Client interface { - GetOffChainData(ctx context.Context, hash common.Hash) (types.ArgBytes, error) - ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash]types.ArgBytes, error) + GetOffChainData(ctx context.Context, hash common.Hash) ([]byte, error) + ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash][]byte, error) SignSequence(signedSequence types.SignedSequence) ([]byte, error) } @@ -68,7 +68,7 @@ func (c *client) SignSequence(signedSequence types.SignedSequence) ([]byte, erro } // GetOffChainData returns data based on it's hash -func (c *client) GetOffChainData(ctx context.Context, hash common.Hash) (types.ArgBytes, error) { +func (c *client) GetOffChainData(ctx context.Context, hash common.Hash) ([]byte, error) { response, err := rpc.JSONRPCCallWithContext(ctx, c.url, "sync_getOffChainData", hash) if err != nil { return nil, err @@ -87,7 +87,7 @@ func (c *client) GetOffChainData(ctx context.Context, hash common.Hash) (types.A } // ListOffChainData returns data based on the given hashes -func (c *client) ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash]types.ArgBytes, error) { +func (c *client) ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash][]byte, error) { response, err := rpc.JSONRPCCallWithContext(ctx, c.url, "sync_listOffChainData", hashes) if err != nil { return nil, err @@ -102,9 +102,14 @@ func (c *client) ListOffChainData(ctx context.Context, hashes []common.Hash) (ma return nil, err } + preparedResult := make(map[common.Hash][]byte) + for key, val := range result { + preparedResult[key] = val + } + fmt.Println("hashes:", hashes) - fmt.Println("result:", string(response.Result)) - fmt.Printf("result: %v\n", result) + fmt.Println("response.Result:", string(response.Result)) + fmt.Printf("preparedResult: %v\n", preparedResult) - return result, nil + return preparedResult, nil } diff --git a/mocks/client.generated.go b/mocks/client.generated.go index 4fc7d8c5..24152c46 100644 --- a/mocks/client.generated.go +++ b/mocks/client.generated.go @@ -26,23 +26,23 @@ func (_m *Client) EXPECT() *Client_Expecter { } // GetOffChainData provides a mock function with given fields: ctx, hash -func (_m *Client) GetOffChainData(ctx context.Context, hash common.Hash) (types.ArgBytes, error) { +func (_m *Client) GetOffChainData(ctx context.Context, hash common.Hash) ([]byte, error) { ret := _m.Called(ctx, hash) if len(ret) == 0 { panic("no return value specified for GetOffChainData") } - var r0 types.ArgBytes + var r0 []byte var r1 error - if rf, ok := ret.Get(0).(func(context.Context, common.Hash) (types.ArgBytes, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, common.Hash) ([]byte, error)); ok { return rf(ctx, hash) } - if rf, ok := ret.Get(0).(func(context.Context, common.Hash) types.ArgBytes); ok { + if rf, ok := ret.Get(0).(func(context.Context, common.Hash) []byte); ok { r0 = rf(ctx, hash) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(types.ArgBytes) + r0 = ret.Get(0).([]byte) } } @@ -74,34 +74,34 @@ func (_c *Client_GetOffChainData_Call) Run(run func(ctx context.Context, hash co return _c } -func (_c *Client_GetOffChainData_Call) Return(_a0 types.ArgBytes, _a1 error) *Client_GetOffChainData_Call { +func (_c *Client_GetOffChainData_Call) Return(_a0 []byte, _a1 error) *Client_GetOffChainData_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Client_GetOffChainData_Call) RunAndReturn(run func(context.Context, common.Hash) (types.ArgBytes, error)) *Client_GetOffChainData_Call { +func (_c *Client_GetOffChainData_Call) RunAndReturn(run func(context.Context, common.Hash) ([]byte, error)) *Client_GetOffChainData_Call { _c.Call.Return(run) return _c } // ListOffChainData provides a mock function with given fields: ctx, hashes -func (_m *Client) ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash]types.ArgBytes, error) { +func (_m *Client) ListOffChainData(ctx context.Context, hashes []common.Hash) (map[common.Hash][]byte, error) { ret := _m.Called(ctx, hashes) if len(ret) == 0 { panic("no return value specified for ListOffChainData") } - var r0 map[common.Hash]types.ArgBytes + var r0 map[common.Hash][]byte var r1 error - if rf, ok := ret.Get(0).(func(context.Context, []common.Hash) (map[common.Hash]types.ArgBytes, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, []common.Hash) (map[common.Hash][]byte, error)); ok { return rf(ctx, hashes) } - if rf, ok := ret.Get(0).(func(context.Context, []common.Hash) map[common.Hash]types.ArgBytes); ok { + if rf, ok := ret.Get(0).(func(context.Context, []common.Hash) map[common.Hash][]byte); ok { r0 = rf(ctx, hashes) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(map[common.Hash]types.ArgBytes) + r0 = ret.Get(0).(map[common.Hash][]byte) } } @@ -133,12 +133,12 @@ func (_c *Client_ListOffChainData_Call) Run(run func(ctx context.Context, hashes return _c } -func (_c *Client_ListOffChainData_Call) Return(_a0 map[common.Hash]types.ArgBytes, _a1 error) *Client_ListOffChainData_Call { +func (_c *Client_ListOffChainData_Call) Return(_a0 map[common.Hash][]byte, _a1 error) *Client_ListOffChainData_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Client_ListOffChainData_Call) RunAndReturn(run func(context.Context, []common.Hash) (map[common.Hash]types.ArgBytes, error)) *Client_ListOffChainData_Call { +func (_c *Client_ListOffChainData_Call) RunAndReturn(run func(context.Context, []common.Hash) (map[common.Hash][]byte, error)) *Client_ListOffChainData_Call { _c.Call.Return(run) return _c } From 2957bc7da28b54b9d39040137a256eed58f3911b Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 13:07:44 -0500 Subject: [PATCH 10/19] Fixed test --- client/client_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 7c5814ba..4198493e 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -104,7 +104,7 @@ func TestClient_GetOffChainData(t *testing.T) { name string hash common.Hash result string - data types.ArgBytes + data []byte statusCode int err error }{ @@ -112,7 +112,7 @@ func TestClient_GetOffChainData(t *testing.T) { name: "successfully got offhcain data", hash: common.BytesToHash([]byte("hash")), result: fmt.Sprintf(`{"result":"%s"}`, hex.EncodeToString([]byte("offchaindata"))), - data: types.ArgBytes("offchaindata"), + data: []byte("offchaindata"), }, { name: "error returned by server", @@ -175,7 +175,7 @@ func TestClient_ListOffChainData(t *testing.T) { name string hashes []common.Hash result string - data map[common.Hash]types.ArgBytes + data map[common.Hash][]byte statusCode int err error }{ @@ -184,7 +184,7 @@ func TestClient_ListOffChainData(t *testing.T) { hashes: []common.Hash{common.BytesToHash([]byte("hash"))}, result: fmt.Sprintf(`{"result":{"%s":"%s"}}`, common.BytesToHash([]byte("hash")).Hex(), hex.EncodeToString([]byte("offchaindata"))), - data: map[common.Hash]types.ArgBytes{ + data: map[common.Hash][]byte{ common.BytesToHash([]byte("hash")): []byte("offchaindata"), }, }, @@ -199,7 +199,7 @@ func TestClient_ListOffChainData(t *testing.T) { hashes: []common.Hash{common.BytesToHash([]byte("hash"))}, result: fmt.Sprintf(`{"result":{"%s":"invalid-signature"}}`, common.BytesToHash([]byte("hash")).Hex()), - data: map[common.Hash]types.ArgBytes{ + data: map[common.Hash][]byte{ common.BytesToHash([]byte("hash")): nil, }, }, From 0abdd61b141356f57a0ff195b31b399c4a9a801f Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 13:15:23 -0500 Subject: [PATCH 11/19] Fixed tests --- client/client.go | 4 ---- test/e2e/e2e_test.go | 9 +++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/client/client.go b/client/client.go index a1f1bfa8..d1dc8d09 100644 --- a/client/client.go +++ b/client/client.go @@ -107,9 +107,5 @@ func (c *client) ListOffChainData(ctx context.Context, hashes []common.Hash) (ma preparedResult[key] = val } - fmt.Println("hashes:", hashes) - fmt.Println("response.Result:", string(response.Result)) - fmt.Printf("preparedResult: %v\n", preparedResult) - return preparedResult, nil } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e9b6733e..d1852258 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -119,6 +119,15 @@ func (tc *testClient) signSequence(t *testing.T, expected *types.SignedSequence, // Check that offchain data has been stored expectedOffchainData := expected.Sequence.OffChainData() + for _, od := range expectedOffchainData { + actualData, err := tc.client.GetOffChainData( + context.Background(), + od.Key, + ) + require.NoError(t, err) + assert.Equal(t, od.Value, actualData) + } + return hashes := make([]common.Hash, len(expectedOffchainData)) for i, od := range expectedOffchainData { hashes[i] = od.Key From fbf08263a61e52f5797e08e75cd95f92cd9db466 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 13:17:42 -0500 Subject: [PATCH 12/19] Fixed tests --- test/e2e/datacommittee_test.go | 8 ++++++-- test/e2e/e2e_test.go | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/e2e/datacommittee_test.go b/test/e2e/datacommittee_test.go index 8742d672..bac871e1 100644 --- a/test/e2e/datacommittee_test.go +++ b/test/e2e/datacommittee_test.go @@ -204,6 +204,12 @@ func TestDataCommittee(t *testing.T) { expectedKeys, err := getSequenceBatchesKeys(clientL1, iter.Event) require.NoError(t, err) for _, m := range membs { + offchainData, err := listOffchainDataKeys(m, expectedKeys) + require.NoError(t, err) + + fmt.Println("expectedKeys:", expectedKeys) + fmt.Println("offchainData:", offchainData) + // Each member (including m0) should have all the keys for _, expected := range expectedKeys { actual, err := getOffchainDataKeys(m, expected) @@ -213,8 +219,6 @@ func TestDataCommittee(t *testing.T) { continue - offchainData, err := listOffchainDataKeys(m, expectedKeys) - require.NoError(t, err) require.Len(t, offchainData, len(expectedKeys)) // Each member (including m0) should have all the keys diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d1852258..7fa19b20 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/ecdsa" "errors" + "fmt" "testing" "github.com/0xPolygon/cdk-data-availability/client" @@ -127,11 +128,14 @@ func (tc *testClient) signSequence(t *testing.T, expected *types.SignedSequence, require.NoError(t, err) assert.Equal(t, od.Value, actualData) } - return + hashes := make([]common.Hash, len(expectedOffchainData)) for i, od := range expectedOffchainData { hashes[i] = od.Key } + fmt.Println("hashes: ", hashes) + + return actualData, err := tc.client.ListOffChainData(context.Background(), hashes) require.NoError(t, err) From b5598596b1fd8cd0e318732a73f240620257f2ad Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 13:26:32 -0500 Subject: [PATCH 13/19] Fixed tests --- db/db.go | 4 ++++ test/e2e/datacommittee_test.go | 9 +++++---- test/e2e/e2e_test.go | 7 +++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/db/db.go b/db/db.go index 0232fb17..71263164 100644 --- a/db/db.go +++ b/db/db.go @@ -221,6 +221,10 @@ func (db *pgDB) GetOffChainData(ctx context.Context, key common.Hash, dbTx sqlx. // ListOffChainData returns values identified by the given keys func (db *pgDB) ListOffChainData(ctx context.Context, keys []common.Hash, dbTx sqlx.QueryerContext) (map[common.Hash]types.ArgBytes, error) { + if len(keys) == 0 { + return nil, nil + } + const listOffchainDataSQL = ` SELECT key, value FROM data_node.offchain_data diff --git a/test/e2e/datacommittee_test.go b/test/e2e/datacommittee_test.go index bac871e1..17b0a870 100644 --- a/test/e2e/datacommittee_test.go +++ b/test/e2e/datacommittee_test.go @@ -206,12 +206,15 @@ func TestDataCommittee(t *testing.T) { for _, m := range membs { offchainData, err := listOffchainDataKeys(m, expectedKeys) require.NoError(t, err) + require.Len(t, offchainData, len(expectedKeys)) - fmt.Println("expectedKeys:", expectedKeys) - fmt.Println("offchainData:", offchainData) + fmt.Println("expectedKeys:", expectedKeys, len(expectedKeys)) + fmt.Println("offchainData:", offchainData, len(offchainData)) // Each member (including m0) should have all the keys for _, expected := range expectedKeys { + fmt.Println("offchainData[expected]:", offchainData[expected]) + actual, err := getOffchainDataKeys(m, expected) require.NoError(t, err) require.Equal(t, expected, actual) @@ -219,8 +222,6 @@ func TestDataCommittee(t *testing.T) { continue - require.Len(t, offchainData, len(expectedKeys)) - // Each member (including m0) should have all the keys for _, expected := range expectedKeys { require.Equal(t, expected, offchainData[expected]) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7fa19b20..67d72272 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -133,12 +133,15 @@ func (tc *testClient) signSequence(t *testing.T, expected *types.SignedSequence, for i, od := range expectedOffchainData { hashes[i] = od.Key } + + actualData, err := tc.client.ListOffChainData(context.Background(), hashes) + require.NoError(t, err) + fmt.Println("hashes: ", hashes) + fmt.Println("actualData: ", actualData) return - actualData, err := tc.client.ListOffChainData(context.Background(), hashes) - require.NoError(t, err) assert.Len(t, actualData, len(expectedOffchainData)) for _, od := range expectedOffchainData { From 5b99fac9f19a7a7f5550a33ce8baa6c458591292 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 13:42:16 -0500 Subject: [PATCH 14/19] Added debug --- test/e2e/datacommittee_test.go | 9 ++++++--- test/e2e/e2e_test.go | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/e2e/datacommittee_test.go b/test/e2e/datacommittee_test.go index 17b0a870..2768ed0d 100644 --- a/test/e2e/datacommittee_test.go +++ b/test/e2e/datacommittee_test.go @@ -206,10 +206,11 @@ func TestDataCommittee(t *testing.T) { for _, m := range membs { offchainData, err := listOffchainDataKeys(m, expectedKeys) require.NoError(t, err) - require.Len(t, offchainData, len(expectedKeys)) - fmt.Println("expectedKeys:", expectedKeys, len(expectedKeys)) - fmt.Println("offchainData:", offchainData, len(offchainData)) + if len(offchainData) != len(expectedKeys) { + fmt.Println("expectedKeys:", expectedKeys, len(expectedKeys)) + fmt.Println("offchainData:", offchainData, len(offchainData)) + } // Each member (including m0) should have all the keys for _, expected := range expectedKeys { @@ -222,6 +223,8 @@ func TestDataCommittee(t *testing.T) { continue + require.Len(t, offchainData, len(expectedKeys)) + // Each member (including m0) should have all the keys for _, expected := range expectedKeys { require.Equal(t, expected, offchainData[expected]) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 67d72272..c76ae616 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -137,8 +137,10 @@ func (tc *testClient) signSequence(t *testing.T, expected *types.SignedSequence, actualData, err := tc.client.ListOffChainData(context.Background(), hashes) require.NoError(t, err) - fmt.Println("hashes: ", hashes) - fmt.Println("actualData: ", actualData) + if len(actualData) != len(expectedOffchainData) { + fmt.Println("hashes: ", hashes) + fmt.Println("actualData: ", actualData) + } return From 6571020827e411f6837cca59ed606a4bd5c47847 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 13:59:05 -0500 Subject: [PATCH 15/19] Added debug --- test/e2e/datacommittee_test.go | 13 ------------- test/e2e/e2e_test.go | 4 ---- 2 files changed, 17 deletions(-) diff --git a/test/e2e/datacommittee_test.go b/test/e2e/datacommittee_test.go index 2768ed0d..1560ec3d 100644 --- a/test/e2e/datacommittee_test.go +++ b/test/e2e/datacommittee_test.go @@ -212,19 +212,6 @@ func TestDataCommittee(t *testing.T) { fmt.Println("offchainData:", offchainData, len(offchainData)) } - // Each member (including m0) should have all the keys - for _, expected := range expectedKeys { - fmt.Println("offchainData[expected]:", offchainData[expected]) - - actual, err := getOffchainDataKeys(m, expected) - require.NoError(t, err) - require.Equal(t, expected, actual) - } - - continue - - require.Len(t, offchainData, len(expectedKeys)) - // Each member (including m0) should have all the keys for _, expected := range expectedKeys { require.Equal(t, expected, offchainData[expected]) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c76ae616..52c2ee77 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -142,10 +142,6 @@ func (tc *testClient) signSequence(t *testing.T, expected *types.SignedSequence, fmt.Println("actualData: ", actualData) } - return - - assert.Len(t, actualData, len(expectedOffchainData)) - for _, od := range expectedOffchainData { assert.Equal(t, od.Value, actualData[od.Key]) } From 1e2302a2598236eb3202c51515f029278842baca Mon Sep 17 00:00:00 2001 From: begmaroman Date: Wed, 20 Mar 2024 14:11:47 -0500 Subject: [PATCH 16/19] Fixed tests --- test/e2e/datacommittee_test.go | 17 ----------------- test/e2e/e2e_test.go | 6 ------ 2 files changed, 23 deletions(-) diff --git a/test/e2e/datacommittee_test.go b/test/e2e/datacommittee_test.go index 1560ec3d..3ec85adc 100644 --- a/test/e2e/datacommittee_test.go +++ b/test/e2e/datacommittee_test.go @@ -207,11 +207,6 @@ func TestDataCommittee(t *testing.T) { offchainData, err := listOffchainDataKeys(m, expectedKeys) require.NoError(t, err) - if len(offchainData) != len(expectedKeys) { - fmt.Println("expectedKeys:", expectedKeys, len(expectedKeys)) - fmt.Println("offchainData:", offchainData, len(offchainData)) - } - // Each member (including m0) should have all the keys for _, expected := range expectedKeys { require.Equal(t, expected, offchainData[expected]) @@ -245,18 +240,6 @@ func getSequenceBatchesKeys(clientL1 *ethclient.Client, event *polygonvalidium.P return keys, err } -func getOffchainDataKeys(m member, tx common.Hash) (common.Hash, error) { - testUrl := fmt.Sprintf("http://127.0.0.1:420%d", m.i) - mc := newTestClient(testUrl, m.addr) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - data, err := mc.client.GetOffChainData(ctx, tx) - if err != nil { - return common.Hash{}, err - } - return crypto.Keccak256Hash(data), nil -} - func listOffchainDataKeys(m member, txes []common.Hash) (map[common.Hash]common.Hash, error) { testUrl := fmt.Sprintf("http://127.0.0.1:420%d", m.i) mc := newTestClient(testUrl, m.addr) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 52c2ee77..82933bed 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ecdsa" "errors" - "fmt" "testing" "github.com/0xPolygon/cdk-data-availability/client" @@ -137,11 +136,6 @@ func (tc *testClient) signSequence(t *testing.T, expected *types.SignedSequence, actualData, err := tc.client.ListOffChainData(context.Background(), hashes) require.NoError(t, err) - if len(actualData) != len(expectedOffchainData) { - fmt.Println("hashes: ", hashes) - fmt.Println("actualData: ", actualData) - } - for _, od := range expectedOffchainData { assert.Equal(t, od.Value, actualData[od.Key]) } From 415bc5264599e433e5e53a61f82544d5f3cc9060 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Thu, 21 Mar 2024 12:53:34 -0500 Subject: [PATCH 17/19] Addressing comments --- db/db_test.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/db/db_test.go b/db/db_test.go index f5c9e921..43700376 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -485,20 +485,41 @@ func Test_DB_ListOffChainData(t *testing.T) { od []types.OffChainData keys []common.Hash expected map[common.Hash]types.ArgBytes + sql string returnErr error }{ { - name: "successfully selected value", + name: "successfully selected one value", + od: []types.OffChainData{{ + Key: common.HexToHash("key1"), + Value: []byte("value1"), + }}, + keys: []common.Hash{ + common.BytesToHash([]byte("key1")), + }, + expected: map[common.Hash]types.ArgBytes{ + common.BytesToHash([]byte("key1")): []byte("value1"), + }, + sql: `SELECT key, value FROM data_node\.offchain_data WHERE key IN \(\$1\)`, + }, + { + name: "successfully selected two values", od: []types.OffChainData{{ Key: common.HexToHash("key1"), Value: []byte("value1"), + }, { + Key: common.HexToHash("key2"), + Value: []byte("value2"), }}, keys: []common.Hash{ common.BytesToHash([]byte("key1")), + common.BytesToHash([]byte("key2")), }, expected: map[common.Hash]types.ArgBytes{ common.BytesToHash([]byte("key1")): []byte("value1"), + common.BytesToHash([]byte("key2")): []byte("value2"), }, + sql: `SELECT key, value FROM data_node\.offchain_data WHERE key IN \(\$1\, \$2\)`, }, { name: "error returned", @@ -509,6 +530,7 @@ func Test_DB_ListOffChainData(t *testing.T) { keys: []common.Hash{ common.BytesToHash([]byte("key1")), }, + sql: `SELECT key, value FROM data_node\.offchain_data WHERE key IN \(\$1\)`, returnErr: errors.New("test error"), }, { @@ -520,6 +542,7 @@ func Test_DB_ListOffChainData(t *testing.T) { keys: []common.Hash{ common.BytesToHash([]byte("underfined")), }, + sql: `SELECT key, value FROM data_node\.offchain_data WHERE key IN \(\$1\)`, returnErr: ErrStateNotSynchronized, }, } @@ -545,7 +568,7 @@ func Test_DB_ListOffChainData(t *testing.T) { preparedKeys[i] = key.Hex() } - expected := mock.ExpectQuery(`SELECT key, value FROM data_node\.offchain_data WHERE key IN \(\$1\)`). + expected := mock.ExpectQuery(tt.sql). WithArgs(preparedKeys...) if tt.returnErr != nil { From 0a36c2aceeb4d2e07f12cde91937afe4a0050666 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Fri, 22 Mar 2024 18:49:46 -0500 Subject: [PATCH 18/19] Empty-Commit From 484cc3791e4dcbd611db0d3df5879432f0cca3a7 Mon Sep 17 00:00:00 2001 From: begmaroman Date: Fri, 22 Mar 2024 19:09:48 -0500 Subject: [PATCH 19/19] Empty-Commit