feat(contract-verifier): Partial matching & automatic verification (#…

…3527)

Fixes #3190
Fixes #3309

I still have to test the migration logic (want to make sure that Era
mainnet data can be migrated correctly & quickly), but the PR is
reviewable otherwise.

This one is much bigger than I indended it to be, sorry 🥲 

- Changes `zksolc` used in contract verifier tests to 1.5.10 (old one
didn't have `ipfs` metadata hash support).
- Fixes problems that occured with newer compilers (e.g. `Yul` bytecode
extraction).
- Makes it easier to run contract verifier tests locally (pins compiler
versions, so that if you have a lot of compilers locally, tests don't
crash).
- Introduces partial matching for contracts
- Reworks the schema so that contracts with matching bytecode can be
"automatically" verified.
- Adds a migration to the new table.

.github/workflows/ci-core-reusable.yml

@@ -72,7 +72,7 @@ jobs:
           ci_run zkstack dev contracts
 
       - name: Download compilers for contract verifier tests
-        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
+        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
 
       - name: Rust unit tests
         run: |
@@ -431,7 +431,7 @@ jobs:
 
       - name: Initialize Contract verifier
         run: |
-          ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
+          ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
           ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
           ci_run zkstack contract-verifier wait --chain era --verbose
 

core/Cargo.toml

@@ -170,6 +170,7 @@ serde = "1"
 serde_json = "1"
 serde_with = "1"
 serde_yaml = "0.9"
+ciborium = "0.2"
 sha2 = "0.10.8"
 sha3 = "0.10.8"
 sqlx = "0.8.1"
@@ -231,7 +232,7 @@ tokio-stream = "0.1.16"
 circuit_encodings = "=0.150.20"
 circuit_sequencer_api = "=0.150.20"
 circuit_definitions = "=0.150.20"
-crypto_codegen = { package = "zksync_solidity_vk_codegen",version = "=0.30.13" }
+crypto_codegen = { package = "zksync_solidity_vk_codegen", version = "=0.30.13" }
 kzg = { package = "zksync_kzg", version = "=0.150.20" }
 zk_evm = { version = "=0.133.0" }
 zk_evm_1_3_1 = { package = "zk_evm", version = "0.131.0-rc.2" }
@@ -240,7 +241,7 @@ zk_evm_1_4_0 = { package = "zk_evm", version = "0.140" }
 zk_evm_1_4_1 = { package = "zk_evm", version = "0.141" }
 zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.20" }
 fflonk = "=0.30.13"
-bellman = {package = "zksync_bellman", version = "=0.30.13"}
+bellman = { package = "zksync_bellman", version = "=0.30.13" }
 
 # New VM; pinned to a specific commit because of instability
 zksync_vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "457d8a7eea9093af9440662e33e598c13ba41633" }

core/bin/contract-verifier/src/main.rs

@@ -6,7 +6,7 @@ use tokio::sync::watch;
 use zksync_config::configs::PrometheusConfig;
 use zksync_contract_verifier_lib::ContractVerifier;
 use zksync_core_leftovers::temp_config_store::{load_database_secrets, load_general_config};
-use zksync_dal::{ConnectionPool, Core};
+use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_queued_job_processor::JobProcessor;
 use zksync_utils::wait_for_tasks::ManagedTasks;
 use zksync_vlog::prometheus::PrometheusExporterConfig;
@@ -25,6 +25,32 @@ struct Opt {
     secrets_path: Option<PathBuf>,
 }
 
+async fn perform_storage_migration(pool: &ConnectionPool<Core>) -> anyhow::Result<()> {
+    const BATCH_SIZE: usize = 1000;
+
+    // Make it possible to override just in case.
+    let batch_size = std::env::var("CONTRACT_VERIFIER_MIGRATION_BATCH_SIZE")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(BATCH_SIZE);
+
+    let mut storage = pool.connection().await?;
+    let migration_performed = storage
+        .contract_verification_dal()
+        .is_verification_info_migration_performed()
+        .await?;
+    if !migration_performed {
+        tracing::info!(batch_size = %batch_size, "Running the storage migration for the contract verifier table");
+        storage
+            .contract_verification_dal()
+            .perform_verification_info_migration(batch_size)
+            .await?;
+    } else {
+        tracing::info!("Storage migration is not needed");
+    }
+    Ok(())
+}
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let opt = Opt::parse();
@@ -51,6 +77,8 @@ async fn main() -> anyhow::Result<()> {
     .build()
     .await?;
 
+    perform_storage_migration(&pool).await?;
+
     let (stop_sender, stop_receiver) = watch::channel(false);
     let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool)
         .await

core/bin/verified_sources_fetcher/src/main.rs

@@ -3,7 +3,7 @@ use std::io::Write;
 use zksync_config::configs::DatabaseSecrets;
 use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_env_config::FromEnv;
-use zksync_types::contract_verification_api::SourceCodeData;
+use zksync_types::contract_verification::api::SourceCodeData;
 
 #[tokio::main]
 async fn main() {

core/lib/basic_types/src/bytecode.rs

@@ -168,6 +168,14 @@ impl BytecodeMarker {
     }
 }
 
+/// Removes padding from the bytecode, if necessary.
+pub fn trim_bytecode(bytecode_hash: BytecodeHash, raw: &[u8]) -> anyhow::Result<&[u8]> {
+    match bytecode_hash.marker() {
+        BytecodeMarker::EraVm => Ok(raw),
+        BytecodeMarker::Evm => trim_padded_evm_bytecode(bytecode_hash, raw),
+    }
+}
+
 /// Removes padding from an EVM bytecode, returning the original EVM bytecode.
 pub fn trim_padded_evm_bytecode(bytecode_hash: BytecodeHash, raw: &[u8]) -> anyhow::Result<&[u8]> {
     if bytecode_hash.marker() != BytecodeMarker::Evm {

core/lib/contract_verifier/src/compilers/mod.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use anyhow::Context as _;
 use serde::{Deserialize, Serialize};
-use zksync_types::contract_verification_api::CompilationArtifacts;
+use zksync_types::contract_verification::api::CompilationArtifacts;
 
 pub(crate) use self::{
     solc::{Solc, SolcInput},

core/lib/contract_verifier/src/compilers/solc.rs

@@ -3,7 +3,7 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio};
 use anyhow::Context;
 use tokio::io::AsyncWriteExt;
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::{
+use zksync_types::contract_verification::api::{
     CompilationArtifacts, SourceCodeData, VerificationIncomingRequest,
 };
 

core/lib/contract_verifier/src/compilers/vyper.rs

@@ -3,7 +3,7 @@ use std::{collections::HashMap, mem, path::PathBuf, process::Stdio};
 use anyhow::Context;
 use tokio::io::AsyncWriteExt;
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::{
+use zksync_types::contract_verification::api::{
     CompilationArtifacts, SourceCodeData, VerificationIncomingRequest,
 };
 

core/lib/contract_verifier/src/compilers/zksolc.rs

@@ -6,7 +6,7 @@ use semver::Version;
 use serde::{Deserialize, Serialize};
 use tokio::io::AsyncWriteExt;
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::{
+use zksync_types::contract_verification::api::{
     CompilationArtifacts, SourceCodeData, VerificationIncomingRequest,
 };
 
@@ -65,6 +65,7 @@ pub(crate) struct Optimizer {
     /// Whether the optimizer is enabled.
     pub enabled: bool,
     /// The optimization mode string.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mode: Option<char>,
 }
 
@@ -144,12 +145,24 @@ impl ZkSolc {
     fn parse_single_file_yul_output(
         output: &str,
     ) -> Result<CompilationArtifacts, ContractVerifierError> {
-        let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap();
-        let cap = re
-            .captures(output)
-            .context("Yul output doesn't match regex")?;
+        let cap = if output.contains("Binary:\n") {
+            // Format of the new output
+            // ======= /tmp/input.yul:Empty =======
+            // Binary:
+            // 00000001002 <..>
+            let re = Regex::new(r"Binary:\n([\da-f]+)").unwrap();
+            re.captures(output)
+                .with_context(|| format!("Yul output doesn't match regex. Output: {output}"))?
+        } else {
+            // Old compiler versions
+            let re_old = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap();
+            re_old
+                .captures(output)
+                .with_context(|| format!("Yul output doesn't match regex. Output: {output}"))?
+        };
         let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str();
         let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?;
+
         Ok(CompilationArtifacts {
             bytecode,
             deployed_bytecode: None,
@@ -255,6 +268,9 @@ impl Compiler<ZkSolcInput> for ZkSolc {
                     .context("cannot create temporary Yul file")?;
                 file.write_all(source_code.as_bytes())
                     .context("failed writing Yul file")?;
+
+                // TODO: `zksolc` support standard JSON for `yul` since 1.5.0, so we don't have
+                // to parse `--bin` output.
                 let child = command
                     .arg(file.path().to_str().unwrap())
                     .arg("--optimization")

core/lib/contract_verifier/src/compilers/zkvyper.rs

@@ -3,7 +3,7 @@ use std::{ffi::OsString, path, path::Path, process::Stdio};
 use anyhow::Context as _;
 use tokio::{fs, io::AsyncWriteExt};
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::CompilationArtifacts;
+use zksync_types::contract_verification::api::CompilationArtifacts;
 
 use super::VyperInput;
 use crate::{

core/lib/contract_verifier/src/lib.rs

@@ -15,9 +15,12 @@ use zksync_dal::{contract_verification_dal::DeployedContractData, ConnectionPool
 use zksync_queued_job_processor::{async_trait, JobProcessor};
 use zksync_types::{
     bytecode::{trim_padded_evm_bytecode, BytecodeHash, BytecodeMarker},
-    contract_verification_api::{
-        self as api, CompilationArtifacts, VerificationIncomingRequest, VerificationInfo,
-        VerificationRequest,
+    contract_verification::{
+        api::{
+            self as api, CompilationArtifacts, VerificationIncomingRequest, VerificationInfo,
+            VerificationProblem, VerificationRequest,
+        },
+        contract_identifier::{ContractIdentifier, Match},
     },
     Address, CONTRACT_DEPLOYER_ADDRESS,
 };
@@ -224,7 +227,7 @@ impl ContractVerifier {
     async fn verify(
         &self,
         mut request: VerificationRequest,
-    ) -> Result<VerificationInfo, ContractVerifierError> {
+    ) -> Result<(VerificationInfo, ContractIdentifier), ContractVerifierError> {
         // Bytecode should be present because it is checked when accepting request.
         let mut storage = self
             .connection_pool
@@ -245,6 +248,8 @@ impl ContractVerifier {
         let bytecode_marker = BytecodeMarker::new(deployed_contract.bytecode_hash)
             .context("unknown bytecode kind")?;
         let artifacts = self.compile(request.req.clone(), bytecode_marker).await?;
+        let identifier =
+            ContractIdentifier::from_bytecode(bytecode_marker, artifacts.deployed_bytecode());
         let constructor_args = match bytecode_marker {
             BytecodeMarker::EraVm => self
                 .decode_era_vm_constructor_args(&deployed_contract, request.req.contract_address)?,
@@ -265,14 +270,28 @@ impl ContractVerifier {
             .context("invalid stored EVM bytecode")?,
         };
 
-        if artifacts.deployed_bytecode() != deployed_bytecode {
-            tracing::info!(
-                request_id = request.id,
-                deployed = hex::encode(deployed_bytecode),
-                compiled = hex::encode(artifacts.deployed_bytecode()),
-                "Deployed (runtime) bytecode mismatch",
-            );
-            return Err(ContractVerifierError::BytecodeMismatch);
+        let mut verification_problems = Vec::new();
+
+        match identifier.matches(deployed_bytecode) {
+            Match::Full => {}
+            Match::Partial => {
+                tracing::trace!(
+                    request_id = request.id,
+                    deployed = hex::encode(deployed_bytecode),
+                    compiled = hex::encode(artifacts.deployed_bytecode()),
+                    "Partial bytecode match",
+                );
+                verification_problems.push(VerificationProblem::IncorrectMetadata);
+            }
+            Match::None => {
+                tracing::trace!(
+                    request_id = request.id,
+                    deployed = hex::encode(deployed_bytecode),
+                    compiled = hex::encode(artifacts.deployed_bytecode()),
+                    "Deployed (runtime) bytecode mismatch",
+                );
+                return Err(ContractVerifierError::BytecodeMismatch);
+            }
         }
 
         match constructor_args {
@@ -284,6 +303,11 @@ impl ContractVerifier {
                         hex::encode(&args),
                         hex::encode(provided_constructor_args)
                     );
+                    // We could, in theory, accept this contract and mark it as partially verified,
+                    // but in during verification it is always possible to reconstruct the
+                    // constructor arguments, so there is no reason for that.
+                    // Mismatching constructor arguments are only needed for "similar bytecodes"
+                    // (e.g. displayed contract as verified without a direct verification request).
                     return Err(ContractVerifierError::IncorrectConstructorArguments);
                 }
             }
@@ -294,11 +318,13 @@ impl ContractVerifier {
 
         let verified_at = Utc::now();
         tracing::trace!(%verified_at, "verified request");
-        Ok(VerificationInfo {
+        let info = VerificationInfo {
             request,
             artifacts,
             verified_at,
-        })
+            verification_problems,
+        };
+        Ok((info, identifier))
     }
 
     async fn compile_zksolc(
@@ -544,17 +570,21 @@ impl ContractVerifier {
     async fn process_result(
         &self,
         request_id: usize,
-        verification_result: Result<VerificationInfo, ContractVerifierError>,
+        verification_result: Result<(VerificationInfo, ContractIdentifier), ContractVerifierError>,
     ) -> anyhow::Result<()> {
         let mut storage = self
             .connection_pool
             .connection_tagged("contract_verifier")
             .await?;
         match verification_result {
-            Ok(info) => {
+            Ok((info, identifier)) => {
                 storage
                     .contract_verification_dal()
-                    .save_verification_info(info)
+                    .save_verification_info(
+                        info,
+                        identifier.bytecode_keccak256,
+                        identifier.bytecode_without_metadata_keccak256,
+                    )
                     .await?;
                 tracing::info!("Successfully processed request with id = {request_id}");
             }

core/lib/contract_verifier/src/resolver/mod.rs

@@ -8,7 +8,7 @@ use std::{
 use anyhow::Context as _;
 use tokio::fs;
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::CompilationArtifacts;
+use zksync_types::contract_verification::api::CompilationArtifacts;
 
 pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver};
 use crate::{