Merge pull request #17 from zkemail/feat/qp-decode

Feat/remove-soft-line-breaks

examples/remove_soft_line_breaks/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "remove_soft_line_breaks"
+type = "bin"
+authors = ["Mach 34"]
+compiler_version = ">=0.35.0"
+
+[dependencies]
+zkemail = { path =  "../../lib"}

examples/remove_soft_line_breaks/src/main.nr

@@ -0,0 +1,64 @@
+use zkemail::{
+    KEY_LIMBS_2048, dkim::RSAPubkey, headers::body_hash::get_body_hash,
+    standard_outputs, Sequence, remove_soft_line_breaks::remove_soft_line_breaks
+};
+use std::hash::sha256_var;
+
+global MAX_EMAIL_HEADER_LENGTH: u32 = 512;
+global MAX_EMAIL_BODY_LENGTH: u32 = 1024;
+
+/**
+ * Verify an arbitrary email signed by a 2048-bit RSA DKIM signature
+ * @dev TOTAL CONSTRAINTS: ~222,783
+ *
+ * @param header - The email header, 0-padded at end to the MAX_EMAIL_HEADER_LENGTH
+ * @param body - The email body, 0-padded at end to the MAX_EMAIL_BODY_LENGTH
+ * @param pubkey - The DKIM RSA Public Key modulus and reduction parameter
+ * @param signature - The DKIM RSA Signature
+ * @param body_hash_index - The index of the body hash in the partial hash array
+ * @param dkim_header_sequence - The index and length of the DKIM header field
+ * @return - 
+ *         0: Pedersen hash of DKIM public key (root of trust)
+ *         1: Pedersen hash of DKIM signature (email nullifier)
+ */
+fn main(
+    header: BoundedVec<u8, MAX_EMAIL_HEADER_LENGTH>,
+    body: BoundedVec<u8, MAX_EMAIL_BODY_LENGTH>,
+    decoded_body: BoundedVec<u8, MAX_EMAIL_BODY_LENGTH>,
+    pubkey: RSAPubkey<KEY_LIMBS_2048>,
+    signature: [Field; KEY_LIMBS_2048],
+    body_hash_index: u32,
+    dkim_header_sequence: Sequence
+) -> pub [Field; 2] {
+    // check the body and header lengths are within bounds
+    assert(header.len() <= MAX_EMAIL_HEADER_LENGTH);
+    assert(body.len() <= MAX_EMAIL_BODY_LENGTH);
+
+    // ~ 86,553 constraints
+    // verify the dkim signature over the header
+    pubkey.verify_dkim_signature(header, signature);
+
+    // ~ 6,289 constraints
+    // extract the body hash from the header
+    let signed_body_hash = get_body_hash(header, dkim_header_sequence, body_hash_index);
+
+    // ~ 113,962 constraints
+    // hash the asserted body
+    let computed_body_hash: [u8; 32] = sha256_var(body.storage(), body.len() as u64);
+
+    // compare the body hashes
+    assert(
+        signed_body_hash == computed_body_hash, "SHA256 hash computed over body does not match body hash found in DKIM-signed header"
+    );
+
+    // ~ 37,982 constraints
+    // ensure the decoded body is the same as the original body
+    assert(
+        remove_soft_line_breaks(body.storage(), decoded_body.storage()),
+        "Decoded body does not properly remove soft line breaks"
+    );
+
+    // ~ 10,255 constraints
+    // hash the pubkey and signature for the standard outputs
+    standard_outputs(pubkey.modulus, signature)
+}

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zk-email/zkemail-nr",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "main": "dist",
   "types": "dist",
   "license": "MIT",
@@ -21,7 +21,7 @@
     "@babel/core": "^7.25.2",
     "@babel/preset-env": "^7.25.4",
     "@babel/preset-typescript": "^7.24.7",
-    "@types/jest": "^29.5.13",
+    "@types/jest": "^29.5.14",
     "@types/mocha": "^10.0.8",
     "@types/node": "^22.5.5",
     "@typescript-eslint/eslint-plugin": "7",

js/src/index.ts

@@ -42,6 +42,8 @@ export type CircuitInput = {
   // inputs used for only masking
   header_mask?: string[];
   body_mask?: string[];
+  // input for decoded body
+  decoded_body?: BoundedVec;
   // inputs used for address extraction
   from_header_sequence?: Sequence;
   from_address_sequence?: Sequence;
@@ -62,33 +64,37 @@ export type InputGenerationArgs = {
   extractTo?: boolean;
 };
 
-// copied without modification, but not publicly exported in original
-// function removeSoftLineBreaks(body: string[]): string[] {
-//   const result = [];
-//   let i = 0;
-//   while (i < body.length) {
-//     if (
-//       i + 2 < body.length &&
-//       body[i] === "61" && // '=' character
-//       body[i + 1] === "13" && // '\r' character
-//       body[i + 2] === "10"
-//     ) {
-//       // '\n' character
-//       // Skip the soft line break sequence
-//       i += 3; // Move past the soft line break
-//     } else {
-//       result.push(body[i]);
-//       i++;
-//     }
-//   }
-//   // Pad the result with zeros to make it the same length as the body
-//   while (result.length < body.length) {
-//     result.push("0");
-//   }
-//   return result;
-// }
-
-// copied without modification, needed for different generateEmailVerifierInnputsFromDKIMResult
+/** Formatted for BoundedVec in case used in other places */
+function removeSoftLineBreaks(body: BoundedVec): BoundedVec {
+  const result = [];
+  let i = 0;
+  let count = 0;
+  while (i < body.storage.length) {
+    if (
+      i + 2 < body.storage.length &&
+      body.storage[i] === "61" && // '=' character
+      body.storage[i + 1] === "13" && // '\r' character
+      body.storage[i + 2] === "10"
+    ) {
+      // '\n' character
+      // Skip the soft line break sequence
+      i += 3; // Move past the soft line break
+    } else {
+      result.push(body.storage[i]);
+      i++;
+      count++;
+    }
+  }
+  // Pad the result with zeros to make it the same length as the body
+  while (result.length < body.storage.length) {
+    result.push("0");
+  }
+  return {
+    storage: result,
+    len: count.toString()
+  };
+}
+
 /**
  * @description Generate circuit inputs for the EmailVerifier circuit from raw email content
  * @param rawEmail Full email content as a buffer or string
@@ -201,6 +207,11 @@ export function generateEmailVerifierInputsFromDKIMResult(
     if (params.headerMask) circuitInputs.header_mask = params.headerMask.map((x) => x.toString());
     if (params.bodyMask) circuitInputs.body_mask = params.bodyMask.map((x) => x.toString());
 
+    // remove soft line breaks
+    if (params.removeSoftLineBreaks) {
+      circuitInputs.decoded_body = removeSoftLineBreaks(circuitInputs.body);
+    }
+
     // address extraction
     if (params.extractFrom) {
       const fromSequences = getAddressHeaderSequence(headers, "from");

js/tests/circuits.test.ts

@@ -1,13 +1,13 @@
 import fs from "fs";
 import path from "path";
 import { ZKEmailProver } from "../src/prover";
-import { generateEmailVerifierInputs } from "../src/index";
-import { makeEmailAddressCharTable, toProverToml } from "../src/utils";
+import { generateEmailVerifierInputs }  from "../src/index";
 // import circuit1024 from "../../examples/verify_email_1024_bit_dkim/target/verify_email_1024_bit_dkim.json";
 import circuit2048 from "../../examples/verify_email_2048_bit_dkim/target/verify_email_2048_bit_dkim.json";
 import circuitPartialHash from "../../examples/partial_hash/target/partial_hash.json";
 import circuitEmailMask from "../../examples/email_mask/target/email_mask.json";
 import circuitExtractAddresses from "../../examples/extract_addresses/target/extract_addresses.json";
+import circuitRemoveSoftLineBreak from "../../examples/remove_soft_line_breaks/target/remove_soft_line_breaks.json";
 
 const emails = {
   small: fs.readFileSync(path.join(__dirname, "./test-data/email-good.eml")),
@@ -23,25 +23,25 @@ const inputParams = {
 };
 
 describe("ZKEmail.nr Circuit Unit Tests", () => {
-  // todo: get a github email from a throwaway account to verify
-  // let prover1024: ZKEmailProver;
-  const selectorText = "All nodes in the Bitcoin network can consult it";
   let prover2048: ZKEmailProver;
   let proverPartialHash: ZKEmailProver;
   let proverMasked: ZKEmailProver;
   let proverExtractAddresses: ZKEmailProver;
+  let proverRemoveSoftLineBreak: ZKEmailProver;
 
   beforeAll(() => {
     //@ts-ignore
     // prover1024 = new ZKEmailProver(circuit1024, "all");
     //@ts-ignore
-    prover2048 = new ZKEmailProver(circuit2048, "all");
+    prover2048 = new ZKEmailProver(circuit2048);
     //@ts-ignore
-    proverPartialHash = new ZKEmailProver(circuitPartialHash, "all");
+    proverPartialHash = new ZKEmailProver(circuitPartialHash);
     //@ts-ignore
-    proverMasked = new ZKEmailProver(circuitEmailMask, "all");
+    proverMasked = new ZKEmailProver(circuitEmailMask);
     //@ts-ignore
-    proverExtractAddresses = new ZKEmailProver(circuitExtractAddresses, "all");
+    proverExtractAddresses = new ZKEmailProver(circuitExtractAddresses);
+    //@ts-ignore
+    proverRemoveSoftLineBreak = new ZKEmailProver(circuitRemoveSoftLineBreak);
   });
 
   afterAll(async () => {
@@ -50,20 +50,21 @@ describe("ZKEmail.nr Circuit Unit Tests", () => {
     await proverPartialHash.destroy();
     await proverMasked.destroy();
     await proverExtractAddresses.destroy();
+    await proverRemoveSoftLineBreak.destroy();
   });
 
-  describe("Successful Cases", () => {
+  describe("Simulate Witnesses", () => {
     it("2048-bit DKIM", async () => {
       const inputs = await generateEmailVerifierInputs(
         emails.small,
         inputParams
       );
       await prover2048.simulateWitness(inputs);
-      console.log(toProverToml(inputs));
+      // console.log(toProverToml(inputs));
     });
     it("Partial Hash", async () => {
       const inputs = await generateEmailVerifierInputs(emails.large, {
-        shaPrecomputeSelector: selectorText,
+        shaPrecomputeSelector: "All nodes in the Bitcoin network can consult it",
         maxHeadersLength: 512,
         maxBodyLength: 192,
       });
@@ -139,5 +140,12 @@ describe("ZKEmail.nr Circuit Unit Tests", () => {
       expect(expectedFrom).toEqual(actualFrom);
       expect(expectedTo).toEqual(actualTo);
     });
+    it("Remove Soft Line Breaks", async () => {
+      const inputs = await generateEmailVerifierInputs(emails.large, {
+        removeSoftLineBreaks: true,
+        ...inputParams,
+      });
+      await proverRemoveSoftLineBreak.simulateWitness(inputs);
+    })
   });
 });

js/tests/proving.test.ts

@@ -7,6 +7,7 @@ import circuit2048 from "../../examples/verify_email_2048_bit_dkim/target/verify
 import circuitPartialHash from "../../examples/partial_hash/target/partial_hash.json";
 import circuitEmailMask from "../../examples/email_mask/target/email_mask.json";
 import circuitExtractAddresses from "../../examples/extract_addresses/target/extract_addresses.json";
+import circuitRemoveSoftLineBreak from "../../examples/remove_soft_line_breaks/target/remove_soft_line_breaks.json";
 
 const emails = {
   small: fs.readFileSync(path.join(__dirname, "./test-data/email-good.eml")),
@@ -22,10 +23,6 @@ const inputParams = {
 };
 
 describe("ZKEmail.nr E2E Tests", () => {
-  // todo: get a github email from a throwaway account to verify
-  // let prover1024: ZKEmailProver;
-  const selectorText = "All nodes in the Bitcoin network can consult it";
-
   describe("2048-bit circuit", () => {
     let prover: ZKEmailProver;
     describe("UltraPlonk", () => {
@@ -95,7 +92,7 @@ describe("ZKEmail.nr E2E Tests", () => {
       });
       it("Partial Hash", async () => {
         const inputs = await generateEmailVerifierInputs(emails.large, {
-          shaPrecomputeSelector: selectorText,
+          shaPrecomputeSelector: "All nodes in the Bitcoin network can consult it",
           maxHeadersLength: 512,
           maxBodyLength: 192,
         });
@@ -114,7 +111,7 @@ describe("ZKEmail.nr E2E Tests", () => {
       });
       it("Partial Hash", async () => {
         const inputs = await generateEmailVerifierInputs(emails.large, {
-          shaPrecomputeSelector: selectorText,
+          shaPrecomputeSelector: "All nodes in the Bitcoin network can consult it",
           maxHeadersLength: 512,
           maxBodyLength: 192,
         });
@@ -222,4 +219,43 @@ describe("ZKEmail.nr E2E Tests", () => {
       });
     });
   });
+  describe("Soft Line Break Removal Circuit", () => {
+    let prover: ZKEmailProver;
+    describe("UltraPlonk", () => {
+      beforeAll(async () => {
+        //@ts-ignore
+        prover = new ZKEmailProver(circuitRemoveSoftLineBreak, "plonk");
+      });
+      afterAll(async () => {
+        prover.destroy();
+      });
+      it("Remmove Soft Line Break", async () => {
+        const inputs = await generateEmailVerifierInputs(emails.large, {
+          removeSoftLineBreaks: true,
+          ...inputParams,
+        });
+        const proof = await prover.fullProve(inputs);
+        const result = await prover.verify(proof);
+        expect(result).toBeTruthy();
+      });
+    });
+    describe("UltraHonk", () => {
+      beforeAll(async () => {
+        //@ts-ignore
+        prover = new ZKEmailProver(circuitRemoveSoftLineBreak, "honk");
+      });
+      afterAll(async () => {
+        prover.destroy();
+      });
+      it("Remove Soft Line Break", async () => {
+        const inputs = await generateEmailVerifierInputs(emails.large, {
+          removeSoftLineBreaks: true,
+          ...inputParams,
+        });
+        const proof = await prover.fullProve(inputs);
+        const result = await prover.verify(proof);
+        expect(result).toBeTruthy();
+      });
+    });
+  });
 });

js/yarn.lock

@@ -1782,10 +1782,10 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
-"@types/jest@^29.5.13":
-  version "29.5.13"
-  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.13.tgz#8bc571659f401e6a719a7bf0dbcb8b78c71a8adc"
-  integrity sha512-wd+MVEZCHt23V0/L642O5APvspWply/rGY5BcW4SUETo2UzPU3Z26qr8jC2qxpimI2jjx9h7+2cj2FwIr01bXg==
+"@types/jest@^29.5.14":
+  version "29.5.14"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.14.tgz#2b910912fa1d6856cadcd0c1f95af7df1d6049e5"
+  integrity sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==
   dependencies:
     expect "^29.0.0"
     pretty-format "^29.0.0"

lib/Nargo.toml

@@ -7,4 +7,5 @@ compiler_version = ">=0.35.0"
 [dependencies]
 rsa = { tag = "v0.3.1", git = "https://github.com/noir-lang/noir_rsa", directory = "lib" }
 base64 = { tag = "v0.2.2", git = "https://github.com/noir-lang/noir_base64" }
+nodash = { tag = "v0.36.0", git = "https://github.com/olehmisar/nodash" }
 # string_search = { tag = "v0.1", git = "https://github.com/noir-lang/noir_string_search"  }

lib/src/lib.nr

@@ -3,8 +3,9 @@ use crate::dkim::RSAPubkey;
 
 pub mod dkim;
 pub mod headers;
-pub mod partial_hash;
 pub mod masking;
+pub mod partial_hash;
+pub mod remove_soft_line_breaks;
 // mod macro;
 mod tests;