refactor(oracle): introduce FeederManager to replace aggregator and caches for transaction handling

.github/workflows/semgrep.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - name: Permission issue fix
         # semgrep for some reason sets the working directory to exocore/exocore
-        run: git config --global --add safe.directory /__w/exocore/exocore
+        run: git config --global --add safe.directory /__w/imuachain/imuachain
       - uses: actions/checkout@v4
       - name: Get Diff
         uses: technote-space/get-diff-action@v6.1.2

.github/workflows/test.yml

@@ -44,3 +44,21 @@ jobs:
         run: |
           make test-unit-cover
         if: env.GIT_DIFF
+  test-unit-e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+          check-latest: true
+      - uses: actions/checkout@v4
+      - uses: technote-space/get-diff-action@v6.1.2
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - name: Test e2e cases
+        run: |
+          make test-unit-e2e
+        if: env.GIT_DIFF

Makefile

@@ -186,7 +186,7 @@ all: build
 
 build-all: tools build lint test vulncheck
 
-.PHONY: distclean clean build-all
+.PHONY: distclean clean build-all build
 
 ###############################################################################
 ###                          makTools & Dependencies                           ###
@@ -305,23 +305,35 @@ test-all: test-unit test-race
 # we want to include all unit tests in the subfolders (tests/e2e/*)
 # We also want to exclude the testutil folder because it contains only
 # helper functions for the tests.
-PACKAGES_UNIT=$(shell go list ./... | grep -v '/tests/e2e$$' | grep -v 'testutil')
+PACKAGES_UNIT=$(shell go list ./... | grep -v '/tests/e2e' | grep -v 'testutil')
+PACKAGES_UNIT_E2E=$(shell go list ./... | grep '/tests/e2e')
 TEST_PACKAGES=./...
-TEST_TARGETS := test-unit test-unit-cover test-race
+TEST_TARGETS := test-unit test-unit-cover test-race test-unit-e2e test-unit-cover-local test-unit-e2e-local
 
 # Test runs-specific rules. To add a new test target, just add
 # a new rule, customise ARGS or TEST_PACKAGES ad libitum, and
 # append the new rule to the TEST_TARGETS list.
-test-unit: ARGS=-timeout=15m -gcflags=all=-l
+test-unit: ARGS=-timeout=15m -gcflags=all=-l --tags devmode
 test-unit: TEST_PACKAGES=$(PACKAGES_UNIT)
 
 test-race: ARGS=-race
 test-race: TEST_PACKAGES=$(PACKAGES_NOSIMULATION)
 $(TEST_TARGETS): run-tests
 
-test-unit-cover: ARGS=-timeout=15m -coverprofile=cover.out -covermode=atomic -gcflags=all=-l
+test-unit-cover: ARGS=-timeout=15m -coverprofile=cover.out -covermode=atomic -gcflags=all=-l --tags devmode
 test-unit-cover: TEST_PACKAGES=$(PACKAGES_UNIT)
 
+test-unit-e2e: ARGS=-timeout=15m --tags devmode
+test-unit-e2e: TEST_PACKAGES=$(PACKAGES_UNIT_E2E)
+
+test-unit-cover-local: ARGS=-timeout=30m -coverprofile=cover.out -covermode=atomic -gcflags=all=-l --tags 'devmode local'
+test-unit-cover-local: TEST_PACKAGES=$(PACKAGES_UNIT)
+
+test-unit-e2e-local: TEST_OPTION=local
+test-unit-e2e-local: ARGS=-timeout=30m --tags devmode
+test-unit-e2e-local: TEST_PACKAGES=$(PACKAGES_UNIT_E2E)
+
+
 test-e2e:
 	@if [ -z "$(TARGET_VERSION)" ]; then \
 		echo "Building docker image from local codebase"; \
@@ -335,9 +347,9 @@ test-e2e:
 
 run-tests:
 ifneq (,$(shell which tparse 2>/dev/null))
-	go test -mod=readonly -json $(ARGS) $(EXTRA_ARGS) $(TEST_PACKAGES) | tparse
+	TEST_OPTION=$(TEST_OPTION) go test -mod=readonly -json $(ARGS) $(EXTRA_ARGS) $(TEST_PACKAGES) | tparse
 else
-	go test -mod=readonly $(ARGS)  $(EXTRA_ARGS) $(TEST_PACKAGES)
+	TEST_OPTION=$(TEST_OPTION) go test -mod=readonly $(ARGS)  $(EXTRA_ARGS) $(TEST_PACKAGES)
 endif
 
 test-import:

app/ante/cosmos/sigverify.go

@@ -391,6 +391,8 @@ func (isd IncrementSequenceDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, sim
 			msg := msg.(*oracletypes.MsgCreatePrice)
 			if accAddress, err := sdk.AccAddressFromBech32(msg.Creator); err != nil {
 				return ctx, errors.New("invalid address")
+				// #nosec G115  // safe conversion
+				// TODO: define msg.Nonce as uint32 to avoid conversion
 			} else if _, err := isd.oracleKeeper.CheckAndIncreaseNonce(ctx, sdk.ConsAddress(accAddress).String(), msg.FeederID, uint32(msg.Nonce)); err != nil {
 				return ctx, err
 			}
@@ -445,6 +447,7 @@ func (vscd ValidateSigCountDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, sim
 	sigCount := 0
 	for _, pk := range pubKeys {
 		sigCount += CountSubKeys(pk)
+		// #nosec G115
 		if uint64(sigCount) > params.TxSigLimit {
 			return ctx, sdkerrors.ErrTooManySignatures.Wrapf("signatures: %d, limit: %d", sigCount, params.TxSigLimit)
 		}

app/ante/cosmos/txsize_gas.go

@@ -76,12 +76,14 @@ func (cgts ConsumeTxSizeGasDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, sim
 			}
 
 			// use stdsignature to mock the size of a full signature
+			// #nosec G115
 			simSig := legacytx.StdSignature{ // nolint:staticcheck // this will be removed when proto is ready
 				Signature: simSecp256k1Sig[:],
 				PubKey:    pubkey,
 			}
 
 			sigBz := legacy.Cdc.MustMarshal(simSig)
+			// #nosec G115
 			cost := sdk.Gas(len(sigBz) + 6)
 
 			// If the pubkey is a multi-signature pubkey, then we estimate for the maximum

app/app.go

@@ -916,14 +916,17 @@ func NewExocoreApp(
 	app.mm.SetOrderBeginBlockers(
 		upgradetypes.ModuleName,    // to upgrade the chain
 		capabilitytypes.ModuleName, // before any module with capabilities like IBC
-		epochstypes.ModuleName,     // to update the epoch
-		feemarkettypes.ModuleName,  // set EIP-1559 gas prices
-		evmtypes.ModuleName,        // stores chain id in memory
-		slashingtypes.ModuleName,   // TODO after reward
-		evidencetypes.ModuleName,   // TODO after reward
-		stakingtypes.ModuleName,    // track historical info
-		ibcexported.ModuleName,     // handles upgrades of chain and hence client
-		authz.ModuleName,           // clear expired approvals
+		// beginblock of oracle will fill params cache, need to be put before epochs will use the params
+		// it need to put before other modules to fill params cahce before access
+		oracleTypes.ModuleName,
+		epochstypes.ModuleName,    // to update the epoch
+		feemarkettypes.ModuleName, // set EIP-1559 gas prices
+		evmtypes.ModuleName,       // stores chain id in memory
+		slashingtypes.ModuleName,  // TODO after reward
+		evidencetypes.ModuleName,  // TODO after reward
+		stakingtypes.ModuleName,   // track historical info
+		ibcexported.ModuleName,    // handles upgrades of chain and hence client
+		authz.ModuleName,          // clear expired approvals
 		// no-op modules
 		ibctransfertypes.ModuleName,
 		icatypes.ModuleName,
@@ -944,7 +947,6 @@ func NewExocoreApp(
 		rewardTypes.ModuleName,
 		exoslashTypes.ModuleName,
 		avsManagerTypes.ModuleName,
-		oracleTypes.ModuleName,
 		distrtypes.ModuleName,
 	)
 

client/docs/swagger-ui/swagger.json

@@ -18574,11 +18574,6 @@
                           "type": "string",
                           "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
                         },
-                        "slash_fraction_miss": {
-                          "type": "string",
-                          "format": "byte",
-                          "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-                        },
                         "slash_fraction_malicious": {
                           "type": "string",
                           "format": "byte",
@@ -19321,11 +19316,6 @@
                                 "type": "string",
                                 "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
                               },
-                              "slash_fraction_miss": {
-                                "type": "string",
-                                "format": "byte",
-                                "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-                              },
                               "slash_fraction_malicious": {
                                 "type": "string",
                                 "format": "byte",
@@ -19693,11 +19683,6 @@
                               "type": "string",
                               "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
                             },
-                            "slash_fraction_miss": {
-                              "type": "string",
-                              "format": "byte",
-                              "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-                            },
                             "slash_fraction_malicious": {
                               "type": "string",
                               "format": "byte",
@@ -19774,6 +19759,11 @@
             "schema": {
               "type": "object",
               "properties": {
+                "version": {
+                  "type": "string",
+                  "format": "int64",
+                  "title": "version of the staker validator list changes"
+                },
                 "staker_info": {
                   "title": "all staker infos under the specified asset",
                   "type": "object",
@@ -19907,6 +19897,11 @@
             "schema": {
               "type": "object",
               "properties": {
+                "version": {
+                  "type": "string",
+                  "format": "int64",
+                  "title": "version of the staker validator list changes"
+                },
                 "staker_infos": {
                   "type": "array",
                   "items": {
@@ -20091,6 +20086,11 @@
             "schema": {
               "type": "object",
               "properties": {
+                "version": {
+                  "type": "string",
+                  "format": "int64",
+                  "title": "version of the staker validator list changes"
+                },
                 "staker_list": {
                   "title": "staker list including all stakers of request asset",
                   "type": "object",
@@ -41249,11 +41249,6 @@
               "type": "string",
               "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
             },
-            "slash_fraction_miss": {
-              "type": "string",
-              "format": "byte",
-              "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-            },
             "slash_fraction_malicious": {
               "type": "string",
               "format": "byte",
@@ -41747,11 +41742,6 @@
                         "type": "string",
                         "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
                       },
-                      "slash_fraction_miss": {
-                        "type": "string",
-                        "format": "byte",
-                        "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-                      },
                       "slash_fraction_malicious": {
                         "type": "string",
                         "format": "byte",
@@ -42234,11 +42224,6 @@
                       "type": "string",
                       "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
                     },
-                    "slash_fraction_miss": {
-                      "type": "string",
-                      "format": "byte",
-                      "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-                    },
                     "slash_fraction_malicious": {
                       "type": "string",
                       "format": "byte",
@@ -42509,11 +42494,6 @@
                   "type": "string",
                   "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
                 },
-                "slash_fraction_miss": {
-                  "type": "string",
-                  "format": "byte",
-                  "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-                },
                 "slash_fraction_malicious": {
                   "type": "string",
                   "format": "byte",
@@ -42529,6 +42509,11 @@
     "exocore.oracle.v1.QueryStakerInfoResponse": {
       "type": "object",
       "properties": {
+        "version": {
+          "type": "string",
+          "format": "int64",
+          "title": "version of the staker validator list changes"
+        },
         "staker_info": {
           "title": "all staker infos under the specified asset",
           "type": "object",
@@ -42599,6 +42584,11 @@
     "exocore.oracle.v1.QueryStakerInfosResponse": {
       "type": "object",
       "properties": {
+        "version": {
+          "type": "string",
+          "format": "int64",
+          "title": "version of the staker validator list changes"
+        },
         "staker_infos": {
           "type": "array",
           "items": {
@@ -42689,6 +42679,11 @@
     "exocore.oracle.v1.QueryStakerListResponse": {
       "type": "object",
       "properties": {
+        "version": {
+          "type": "string",
+          "format": "int64",
+          "title": "version of the staker validator list changes"
+        },
         "staker_list": {
           "title": "staker list including all stakers of request asset",
           "type": "object",
@@ -43048,11 +43043,6 @@
                   "type": "string",
                   "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
                 },
-                "slash_fraction_miss": {
-                  "type": "string",
-                  "format": "byte",
-                  "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-                },
                 "slash_fraction_malicious": {
                   "type": "string",
                   "format": "byte",
@@ -43120,11 +43110,6 @@
           "type": "string",
           "title": "oracle_malicious_jail_duration defines the duratin one validator should be jailed for malicious behavior"
         },
-        "slash_fraction_miss": {
-          "type": "string",
-          "format": "byte",
-          "title": "slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price"
-        },
         "slash_fraction_malicious": {
           "type": "string",
           "format": "byte",

precompiles/avs/events.go

@@ -271,6 +271,8 @@ func (p Precompile) EmitTaskSubmittedByOperator(ctx sdk.Context, stateDB vm.Stat
 	}
 	// Prepare the event data:sender,TaskResponse, BlsSignature, Phase
 	arguments := abi.Arguments{event.Inputs[2], event.Inputs[3], event.Inputs[4], event.Inputs[5]}
+	// #nosec G115
+	// TODO: consider modify define of Phase to uint8
 	packed, err := arguments.Pack(params.CallerAddress.String(), params.TaskResponse, params.BlsSignature, uint8(params.Phase))
 	if err != nil {
 		return err

precompiles/delegation/tx.go

@@ -52,7 +52,6 @@ func (p Precompile) Delegate(
 	if err != nil {
 		return nil, err
 	}
-
 	err = p.delegationKeeper.DelegateTo(ctx, delegationParams)
 	if err != nil {
 		return nil, err

proto/exocore/oracle/v1/genesis.proto

@@ -55,6 +55,8 @@ message StakerInfosAssets {
   string asset_id = 1;
   // stakerInfos
   repeated StakerInfo staker_infos = 2;
+  // nst_version is the version of nst to track validator list changes
+  int64 nst_version = 3;
 }
 
 // stakerListAssets bond stakerList to their related assets id
@@ -63,6 +65,8 @@ message StakerListAssets {
   string asset_id = 1;
   // stakerList
   StakerList staker_list = 2;
+  // nst_version is the version of nst to track validator list changes
+  int64 nst_version = 3;
 }
 
 // ValidatorMissedRounds record missed rounds indexes for a validator which consAddr corresponding to the address

proto/exocore/oracle/v1/params.proto

@@ -50,7 +50,7 @@
 }
 
 // slashing related params
 message SlashingParams {
   // reported_rounds_window defines how many rounds included in one window for performance review of missing report
   int64 reported_rounds_window = 1;
   // min_reported_perwindow defines at least how many rounds should be reported, this is a percentage of window
@@ -71,14 +71,8 @@
     (amino.dont_omitempty) = true,
     (gogoproto.stdduration) = true
   ];
-  // slash_fraction_miss defines the fraction one validator should be punished for msissing reporting price
-  bytes slash_fraction_miss = 5 [
-    (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
-    (gogoproto.nullable) = false,
-    (amino.dont_omitempty) = true
-  ];
   // slash_fraction_malicious defines the fraction one validator should be punished for malicious behavior
-  bytes slash_fraction_malicious = 6 [
+  bytes slash_fraction_malicious = 5 [
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
     (gogoproto.nullable) = false,
     (amino.dont_omitempty) = true