Repo sync (#170)

secretflow · Dec 8, 2023 · d9f18c7 · d9f18c7
1 parent 8492bdf
commit d9f18c7
Show file tree

Hide file tree

Showing 50 changed files with 2,298 additions and 612 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -31,6 +31,9 @@ build:macos --host_copt=-Wa,--noexecstack
 # platform specific config
 # Bazel will automatic pick platform config since we have enable_platform_specific_config set
 build:macos --features=-supports_dynamic_linker
+build:macos --linkopt="-Wl,-no_warn_duplicate_libraries"
+build:macos --copt=-Wno-unused-command-line-argument
+build:macos --host_copt=-Wno-unused-command-line-argument
 
 build:asan --features=asan
 build:ubsan --features=ubsan

diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-6.2.1
+6.4.0
diff --git a/.circleci/continue-config.yml b/.circleci/continue-config.yml
@@ -58,7 +58,7 @@ jobs:
           path: test_logs.tar.gz
   macOS_ut_arm64:
     macos:
-      xcode: 14.2
+      xcode: 15.1
     environment:
       HOMEBREW_NO_AUTO_UPDATE: 1
     resource_class: macos.m1.medium.gen1

diff --git a/ALGORITHMS.md b/ALGORITHMS.md
@@ -0,0 +1,33 @@
+# Supported Crypto Algorithms
+
+TODO
+
+## Primitives
+- OT
+    - Simplest OT : https://eprint.iacr.org/2015/267.pdf
+    - INKP OT Extension : https://www.iacr.org/archive/crypto2003/27290145/27290145.pdf
+    - KOS OT Extension : https://eprint.iacr.org/2015/546.pdf
+    - KKRT OT Extension : https://eprint.iacr.org/2016/799.pdf
+    - SGRR OT Extension: https://eprint.iacr.org/2019/1084.pdf
+    - GYWZ OT Extension : https://eprint.iacr.org/2022/1431.pdf
+    - Ferret OT Extension : https://eprint.iacr.org/2020/924.pdf
+    - Softspoken OT Extension : https://eprint.iacr.org/2022/192.pdf
+- VOLE(over f2k)
+    - base VOLE : https://eprint.iacr.org/2016/505.pdf
+    - Silent VOLE : https://eprint.iacr.org/2019/1159.pdf, https://eprint.iacr.org/2021/1150.pdf, https://eprint.iacr.org/2022/1014.pdf
+
+## Theoretical Tools
+
+- Random Oracle
+- Random Permutation
+- Local Linear Code :  https://eprint.iacr.org/2020/924.pdf
+- Low Density Parity Check Code (Silver Code) : https://eprint.iacr.org/2021/1150.pdf
+- Expanding Accumulation Code : https://eprint.iacr.org/2022/1014.pdf
+- Correlation-Robust Hash Function : https://eprint.iacr.org/2019/074.pdf
+- Circular Correlation-Robust Hash Function : https://eprint.iacr.org/2019/074.pdf
+
+## Basic (Traditional) algorithms
+
+- AES
+- Hash: SHA2, SM2
+- RSA
diff --git a/README.md b/README.md
@@ -15,26 +15,24 @@ Repo layout:
 - [io](yacl/io/): a simple streaming-based io library.
 - [link](yacl/link/): a simple rpc-based MPI framework, providing the [SPMD](https://en.wikipedia.org/wiki/SPMD) parallel programming capability.
 
-## Supported Crypto Primitives
+## Supported Crypto Algorithms
 
-Oblivious Transfer (and extensions)
+See **Full List** of supported algorithms: [ALGORITHMS.md](ALGORITHMS.md)
 
-- [Simplest OT](https://eprint.iacr.org/2015/267.pdf): 1-out-of-2 OT
-- [IKNP OTe](https://www.iacr.org/archive/crypto2003/27290145/27290145.pdf): 1-out-of-2 OT extension
-- [Ferret OTe](https://eprint.iacr.org/2020/924): 1-out-of-2 OT extension
-- [KKRT OTe](https://eprint.iacr.org/2016/799.pdf): 1-out-of-n OT (a.k.a OPRF)
-- [SGRR OTe](https://eprint.iacr.org/2019/1084.pdf): (n-1)-out-of-n OTe
-- [GYWZ+ OTe](https://eprint.iacr.org/2022/1431.pdf): (n-1)-out-of-n OTe with correlated GGM tree optimizations
+**Selected algorithms**:
 
-Distributed Point Function
+- Oblivious Transfer (and extensions): [Simplest OT](https://eprint.iacr.org/2015/267.pdf), [IKNP OTe](https://www.iacr.org/archive/crypto2003/27290145/27290145.pdf), [Ferret OTe](https://eprint.iacr.org/2020/924), [KKRT OTe](https://eprint.iacr.org/2016/799.pdf), [SGRR OTe](https://eprint.iacr.org/2019/1084.pdf).
+- VOLE: [Silent VOLE](https://eprint.iacr.org/2019/1159.pdf), [Sparse VOLE (GF128)](https://eprint.iacr.org/2019/1084.pdf)
+- Distributed Point Function: [BGI16](https://eprint.iacr.org/2018/707.pdf)
+- Threshold Proxy-Re-encryption: [umbral with GM](https://github.com/nucypher/umbral-doc/blob/master/umbral-doc.pdf).
 
-- [BGI16](https://eprint.iacr.org/2018/707.pdf)
-
-Threshold Proxy-Re-encryption
+## Build
 
-- A substitute of [umbral](https://github.com/nucypher/umbral-doc/blob/master/umbral-doc.pdf). Our implementation supports SM2, SM3 and SM4.
+### Supported platforms
 
-## Build
+|     | Linux x86_64 | Linux aarch64 | macOS x86_64 | macOS Apple Silicon | Windows x86_64 | Windows WSL2 x86_64 |
+|-----|--------------|---------------|--------------|---------------------|----------------|---------------------|
+| CPU | yes          | yes           | yes          | yes                 | no             | yes                 |
 
 ### Prerequisite
 

diff --git a/bazel/patches/libtommath.patch b/bazel/patches/libtommath.patch
@@ -1,5 +1,5 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 8f85249..53e0365 100644
+index dfbcb0f..72f9a46 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
 @@ -113,7 +113,7 @@ set_target_properties(${PROJECT_NAME} PROPERTIES
@@ -9,28 +9,18 @@ index 8f85249..53e0365 100644
 -    PUBLIC_HEADER "${PUBLIC_HEADERS}"
 +    PUBLIC_HEADER "${HEADERS}"
  )
- 
+
  option(COMPILE_LTO "Build with LTO enabled")
 diff --git a/tommath_private.h b/tommath_private.h
-index d88d263..46caa96 100644
+index d319a1d..5f4446e 100644
 --- a/tommath_private.h
 +++ b/tommath_private.h
-@@ -188,14 +188,14 @@ MP_STATIC_ASSERT(prec_geq_min_prec, MP_DEFAULT_DIGIT_COUNT >= MP_MIN_DIGIT_COUNT
+@@ -17,7 +17,7 @@
+  * On Win32 a .def file must be used to specify the exported symbols.
+  */
+ #if defined(__GNUC__) && __GNUC__ >= 4 && !defined(_WIN32) && !defined(__CYGWIN__)
+-#   define MP_PRIVATE __attribute__ ((visibility ("hidden")))
++#   define MP_PRIVATE
+ #else
+ #   define MP_PRIVATE
  #endif
-
- /* random number source */
--extern MP_PRIVATE mp_err(*s_mp_rand_source)(void *out, size_t size);
-+extern mp_err(*s_mp_rand_source)(void *out, size_t size);
-
- /* lowlevel functions, do not call! */
- MP_PRIVATE bool s_mp_get_bit(const mp_int *a, int b) MP_WUR;
- MP_PRIVATE int s_mp_log_2expt(const mp_int *a, mp_digit base) MP_WUR;
-
- MP_PRIVATE mp_err s_mp_add(const mp_int *a, const mp_int *b, mp_int *c) MP_WUR;
--MP_PRIVATE mp_err s_mp_div_3(const mp_int *a, mp_int *c, mp_digit *d) MP_WUR;
-+mp_err s_mp_div_3(const mp_int *a, mp_int *c, mp_digit *d) MP_WUR;
- MP_PRIVATE mp_err s_mp_div_recursive(const mp_int *a, const mp_int *b, mp_int *q, mp_int *r) MP_WUR;
- MP_PRIVATE mp_err s_mp_div_school(const mp_int *a, const mp_int *b, mp_int *c, mp_int *d) MP_WUR;
- MP_PRIVATE mp_err s_mp_div_small(const mp_int *a, const mp_int *b, mp_int *c, mp_int *d) MP_WUR;
-
-
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -312,15 +312,15 @@ def _com_github_libtom_libtommath():
     maybe(
         http_archive,
         name = "com_github_libtom_libtommath",
-        sha256 = "da0759723645d974b82f134a26a1933a08fee887580132f55482c606ec688188",
+        sha256 = "dbfdafbaeb51ff92fdd3f2505ec0490f8a9badc2a71b378219856b68d470f0aa",
         type = "tar.gz",
-        strip_prefix = "libtommath-7f96509df1a6b44867bbda56bbf2cb92524be8ef",
+        strip_prefix = "libtommath-8ce69f7b5e2f34620633f4fb5c231045a8dc2f54",
         patch_args = ["-p1"],
         patches = [
             "@yacl//bazel:patches/libtommath.patch",
         ],
         urls = [
-            "https://github.com/libtom/libtommath/archive/7f96509df1a6b44867bbda56bbf2cb92524be8ef.tar.gz",
+            "https://github.com/libtom/libtommath/archive/8ce69f7b5e2f34620633f4fb5c231045a8dc2f54.tar.gz",
         ],
         build_file = "@yacl//bazel:libtommath.BUILD",
     )

diff --git a/bazel/yacl.bzl b/bazel/yacl.bzl
@@ -39,12 +39,22 @@ AES_COPT_FLAGS = select({
     ],
 })
 
-OMP_LINK_FLAGS = select({
+OMP_DEPS = select({
     "@bazel_tools//src/conditions:darwin_x86_64": ["@macos_omp_x64//:openmp"],
     "@bazel_tools//src/conditions:darwin_arm64": ["@macos_omp_arm64//:openmp"],
     "//conditions:default": [],
 })
 
+OMP_CFLAGS = select({
+    "@platforms//os:macos": ["-Xclang", "-fopenmp"],
+    "//conditions:default": ["-fopenmp"],
+})
+
+OMP_LINKFLAGS = select({
+    "@platforms//os:macos": [],
+    "//conditions:default": ["-fopenmp"],
+})
+
 def _yacl_copts():
     return select({
         "@yacl//bazel:yacl_build_as_release": RELEASE_FLAGS,
@@ -86,14 +96,11 @@ def yacl_configure_make(**attrs):
 def yacl_cc_test(
         copts = [],
         deps = [],
-        linkstatic = True,
         **kwargs):
     cc_test(
         copts = _yacl_copts() + copts,
         deps = deps + [
             "@com_google_googletest//:gtest_main",
         ],
-        # static link for tcmalloc
-        linkstatic = True,
         **kwargs
     )
diff --git a/yacl/base/buffer_test.cc b/yacl/base/buffer_test.cc
@@ -14,6 +14,7 @@
 
 #include "yacl/base/buffer.h"
 
+#include <cstdint>
 #include <vector>
 
 #include "gtest/gtest.h"
@@ -25,8 +26,8 @@ namespace yacl::test {
 TEST(BufferTest, ParallelWorks) {
   std::vector<Buffer> v;
   v.resize(100000);
-  parallel_for(0, v.size(), 1, [&](int64_t beg, int64_t end) {
-    for (int64_t i = beg; i < end; ++i) {
+  parallel_for(0, v.size(), [&](int64_t begin, int64_t end) {
+    for (int64_t i = begin; i < end; ++i) {
       v[i] = Buffer(fmt::format("hello_{}", i));
     }
   });

diff --git a/yacl/base/exception.h b/yacl/base/exception.h
@@ -283,7 +283,7 @@ class EnforceNotMet : public Exception {
   do {                                                                   \
     if (!(condition)) {                                                  \
       ::yacl::stacktrace_t __stacks__;                                   \
-      int __dep__ = absl::GetStackTrace(                                 \
+      const int __dep__ = absl::GetStackTrace(                           \
           __stacks__.data(), ::yacl::internal::kMaxStackTraceDep, 0);    \
       throw ::yacl::EnforceNotMet(__FILE__, __LINE__, #condition,        \
                                   ::yacl::internal::Format(__VA_ARGS__), \
@@ -414,4 +414,10 @@ T CheckNotNull(T t) {
   return t;
 }
 
+#ifdef NDEBUG
+#define WEAK_ENFORCE(condition, ...) ((void)0)
+#else
+#define WEAK_ENFORCE(condition, ...) YACL_ENFORCE(condition, __VA_ARGS__)
+#endif
+
 }  // namespace yacl
diff --git a/yacl/crypto/base/ecc/ecc_test.cc b/yacl/crypto/base/ecc/ecc_test.cc
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
 #include <cstdio>
 #include <mutex>
 #include <random>
@@ -289,7 +290,7 @@ class EcCurveTest : public ::testing::TestWithParam<std::string> {
     constexpr int64_t ts = 1 << 15;
     std::array<EcPoint, ts> buf;
     auto g = ec_->GetGenerator();
-    yacl::parallel_for(0, ts, 1, [&](int64_t beg, int64_t end) {
+    yacl::parallel_for(0, ts, [&](int64_t beg, int64_t end) {
       auto point = ec_->MulBase(MPInt(beg));
       buf[beg] = point;
       for (int64_t i = beg + 1; i < end; ++i) {
@@ -392,7 +393,7 @@ TEST(OpensslMemLeakTest, DISABLED_MulBaseLeaks) {
       EcGroupFactory::Instance().Create("sm2", ArgLib = "openssl");
 
   std::mutex mutex;
-  yacl::parallel_for(0, 2, 1, [&](int64_t, int64_t) {
+  yacl::parallel_for(0, 2, [&](int64_t, int64_t) {
     std::lock_guard<std::mutex> guard(mutex);
     // memory leaks here even with serial calls.
     ec->MulBase(0_mp);

diff --git a/yacl/crypto/base/ecc/openssl/openssl_test.cc b/yacl/crypto/base/ecc/openssl/openssl_test.cc
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
+
 #include "gtest/gtest.h"
 
 #include "yacl/crypto/base/ecc/openssl/openssl_group.h"
@@ -103,7 +105,7 @@ TEST(OpensslMemLeakTest, MulBaseLeaks) {
       yacl::crypto::EcGroupFactory::Instance().Create("sm2",
                                                       ArgLib = "openssl");
 
-  yacl::parallel_for(0, 2, 1, [&](int64_t, int64_t) {
+  yacl::parallel_for(0, 2, [&](int64_t, int64_t) {
     // no memory leak here, but the same code in ecc_test.cc leaks.
     ec->MulBase(0_mp);
   });

diff --git a/yacl/crypto/base/ecc/pairing_test.cc b/yacl/crypto/base/ecc/pairing_test.cc
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
+
 #include "fmt/ranges.h"
 #include "gtest/gtest.h"
 
@@ -230,12 +232,14 @@ class PairingCurveTest : public ::testing::TestWithParam<std::string> {
     constexpr int64_t ts = 1 << 15;
     std::array<EcPoint, ts> buf;
     auto g = ec->GetGenerator();
-    yacl::parallel_for(0, ts, 1, [&](int64_t beg, int64_t end) {
-      auto point = ec->MulBase(MPInt(beg));
-      buf[beg] = point;
-      for (int64_t i = beg + 1; i < end; ++i) {
-        point = ec->Add(point, g);
-        buf[i] = point;
+    yacl::parallel_for(0, ts, [&](int64_t beg, int64_t end) {
+      for (int64_t i = beg; i < end; ++i) {
+        auto point = ec->MulBase(MPInt(beg));
+        buf[beg] = point;
+        for (int64_t i = beg + 1; i < end; ++i) {
+          point = ec->Add(point, g);
+          buf[i] = point;
+        }
       }
     });
 
@@ -326,14 +330,12 @@ TEST(Pairing_Multi_Instance_Test, Works) {
     // TODO: temporarily disable mcl pairing-related test, since its weird error
     // on Intel Mac
     if (lib_name != "libmcl") {
-      yacl::parallel_for(0, 10, 1, [&](int64_t x, int64_t y) {
-        for (int64_t i = x; i < y; i++) {
-          std::shared_ptr<PairingGroup> pairing =
-              PairingGroupFactory::Instance().Create(pairing_name,
-                                                     ArgLib = lib_name);
-          pairing->Pairing(pairing->GetG1()->GetGenerator(),
-                           pairing->GetG2()->GetGenerator());
-        }
+      yacl::parallel_for(0, 10, [&](int64_t, int64_t) {
+        std::shared_ptr<PairingGroup> pairing =
+            PairingGroupFactory::Instance().Create(pairing_name,
+                                                   ArgLib = lib_name);
+        pairing->Pairing(pairing->GetG1()->GetGenerator(),
+                         pairing->GetG2()->GetGenerator());
       });
     }
   }

diff --git a/yacl/crypto/tools/BUILD.bazel b/yacl/crypto/tools/BUILD.bazel
@@ -84,7 +84,6 @@ yacl_cc_library(
         ":code_interface",
         "//yacl/crypto/tools:random_permutation",
         "//yacl/math:gadget",
-        "//yacl/utils:thread_pool",
     ] + select({
         "@platforms//cpu:aarch64": [
             "@com_github_dltcollab_sse2neon//:sse2neon",
@@ -109,8 +108,8 @@ yacl_cc_library(
     deps = [
         ":code_interface",
         "//yacl/base:block",
+        "//yacl/base:exception",
         "//yacl/base:int128",
-        "//yacl/utils:thread_pool",
     ] + select({
         "@platforms//cpu:aarch64": [
             "@com_github_dltcollab_sse2neon//:sse2neon",
@@ -136,7 +135,6 @@ yacl_cc_library(
         ":linear_code",
         "//yacl/base:block",
         "//yacl/base:int128",
-        "//yacl/utils:thread_pool",
     ] + select({
         "@platforms//cpu:aarch64": [
             "@com_github_dltcollab_sse2neon//:sse2neon",

diff --git a/yacl/crypto/tools/linear_code.h b/yacl/crypto/tools/linear_code.h
@@ -16,7 +16,6 @@
 
 #include <algorithm>
 #include <array>
-#include <vector>
 
 #include "absl/types/span.h"