Merge branch 'master' into di-serializer-copy

.github/workflows/nix.yml

@@ -0,0 +1,55 @@
+name: Test Nix
+
+on:
+  push:
+    branches:
+      - "**"
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - os: darwin
+            cpu: x86_64
+            base: macos-13 # always x86_64-darwin
+          - os: darwin
+            cpu: arm64
+            base: macos-14 # always arm64-darwin
+          - os: linux
+            cpu: x86_64
+            base: ubuntu-24.04 # always x86_64-linux-gnu
+          - os: linux
+            cpu: aarch64
+            base: arm-4core-linux-ubuntu24.04 # always aarch64-linux-gnu
+        nix:
+          - 24.05
+
+    name: Test Nix (${{ matrix.platform.cpu }}-${{ matrix.platform.os }}, ${{ matrix.nix }})
+    runs-on: ${{ matrix.platform.base }}
+
+    permissions:
+      contents: read
+      id-token: write
+
+    env:
+      SKIP_SIMPLECOV: 1
+      DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
+      DD_REMOTE_CONFIGURATION_ENABLED: false
+
+    steps:
+      - name: Check CPU arch
+        run: |
+          test "$(uname -m)" = "${{ matrix.platform.cpu }}"
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Print ruby version
+        run: |
+          nix develop --command which ruby
+          nix develop --command ruby --version
+      - name: Bundle install
+        run: nix develop --command bundle install
+      - name: Run spec:main
+        run: nix develop --command bundle exec rake spec:main

.github/workflows/system-tests.yml

@@ -12,8 +12,8 @@ env:
   REGISTRY: ghcr.io
   REPO: ghcr.io/datadog/dd-trace-rb
   ST_REF: main
-  FORCE_TESTS: -F tests/appsec/waf/test_addresses.py::Test_GraphQL -F tests/appsec/test_blocking_addresses.py::Test_BlockingGraphqlResolvers
-  FORCE_TESTS_SCENARIO: GRAPHQL_APPSEC
+  FORCE_TESTS: -F tests/appsec/test_asm_standalone.py
+  FORCE_TESTS_SCENARIO: APPSEC_STANDALONE
 
 jobs:
   build-harness:
@@ -199,6 +199,7 @@ jobs:
           - APPSEC_DISABLED
           - APPSEC_BLOCKING_FULL_DENYLIST
           - APPSEC_REQUEST_BLOCKING
+          - APPSEC_STANDALONE
         include:
           - library: ruby
             app: rack

.github/workflows/test-yjit.yaml

@@ -15,6 +15,10 @@ jobs:
           - '--yjit'
           - '--yjit --yjit-stats=quiet'
         exclude:
+          # NOTE: `--yjit-stats=quiet` is only present in 3.3+.
+          #   We've had `--yjit-stats` since 3.0, but
+          #   1) it dumps stat info to stdout, and
+          #   2) none of our <3.3 tests require the additional stats
           - ruby: '3.2'
             rubyopt: '--yjit --yjit-stats=quiet'
     name: Test YJIT (${{ matrix.os }}, ${{ matrix.ruby }} ${{ matrix.rubyopt }})

.gitignore

@@ -38,7 +38,8 @@ ext/**/skipped_reason.txt
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
 # Ignore local variables
-.envrc
+/.envrc
+/.direnv
 
 # lock files
 Gemfile.lock

.gitlab/install_datadog_deps.rb

@@ -70,7 +70,7 @@
 }
 
 [
-  'debase-ruby_core_source',
+  'datadog-ruby_core_source',
   'ffi',
   'libddwaf',
   'msgpack',

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+* AppSec: Add Experimental Standalone AppSec Threats billing ([#3965][])
+
 ## [2.4.0] - 2024-10-11
 
 ### Added

datadog.gemspec

@@ -64,7 +64,7 @@ Gem::Specification.new do |spec|
   # Used by the profiler native extension to support Ruby < 2.6 and > 3.2
   #
   # We decided to pin it at the latest available version and will manually bump the dependency as needed.
-  spec.add_dependency 'debase-ruby_core_source', '= 3.3.1'
+  spec.add_dependency 'datadog-ruby_core_source', '= 3.3.6'
 
   # Used by appsec
   spec.add_dependency 'libddwaf', '~> 1.14.0.0.0'

default.nix

@@ -0,0 +1,11 @@
+# flake-compat shim for usage without flakes
+(import
+  (
+    let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
+    fetchTarball {
+      url = lock.nodes.flake-compat.locked.url or "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+      sha256 = lock.nodes.flake-compat.locked.narHash;
+    }
+  )
+  { src = ./.; }
+).defaultNix

ext/datadog_profiling_native_extension/NativeExtensionDesign.md

@@ -68,7 +68,7 @@ internal types, structures and functions).
 Because these private header files are not included in regular Ruby installations, we have two different workarounds:
 
 1. for Ruby versions 2.6 to 3.2 we make use use the Ruby private MJIT header
-2. for Ruby versions < 2.6 and > 3.2 we make use of the `debase-ruby_core_source` gem
+2. for Ruby versions < 2.6 and > 3.2 we make use of the `datadog-ruby_core_source` gem
 
 Functions which make use of these headers are defined in the <private_vm_api_acccess.c> file.
 
@@ -91,9 +91,9 @@ version. e.g. `rb_mjit_min_header-2.7.4.h`.
 
 This header was removed in Ruby 3.3.
 
-### Approach 2: Using the `debase-ruby_core_source` gem
+### Approach 2: Using the `datadog-ruby_core_source` gem
 
-The [`debase-ruby_core_source`](https://github.com/ruby-debug/debase-ruby_core_source) contains almost no code;
+The [`datadog-ruby_core_source`](https://github.com/DataDog/datadog-ruby_core_source) contains almost no code;
 instead, it just contains per-Ruby-version folders with the private VM headers (`.h`) files for that version.
 
 Thus, even though a regular Ruby installation does not include these files, we can access the copy inside this gem.

ext/datadog_profiling_native_extension/extconf.rb

@@ -256,21 +256,21 @@ def skip_building_extension!(reason)
   create_makefile EXTENSION_NAME
 else
   # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  # the debase-ruby_core_source gem to get access to private VM headers.
+  # the datadog-ruby_core_source gem to get access to private VM headers.
   # This gem ships source code copies of these VM headers for the different Ruby VM versions;
-  # see https://github.com/ruby-debug/debase-ruby_core_source for details
+  # see https://github.com/DataDog/datadog-ruby_core_source for details
 
   create_header
 
-  require "debase/ruby_core_source"
+  require "datadog/ruby_core_source"
   dir_config("ruby") # allow user to pass in non-standard core include directory
 
   # This is a workaround for a weird issue...
   #
-  # The mkmf tool defines a `with_cppflags` helper that debase-ruby_core_source uses. This helper temporarily
+  # The mkmf tool defines a `with_cppflags` helper that datadog-ruby_core_source uses. This helper temporarily
   # replaces `$CPPFLAGS` (aka the C pre-processor [not c++!] flags) with a different set when doing something.
   #
-  # The debase-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
+  # The datadog-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
   # path. But because `with_cppflags` replaces `$CPPFLAGS`, well, the default `$CPPFLAGS` are not included in the
   # makefile.
   #
@@ -281,12 +281,12 @@ def skip_building_extension!(reason)
   # `VM_CHECK_MODE=1` when building Ruby will trigger this issue (because somethings in structures the profiler reads
   # are ifdef'd out using this setting).
   #
-  # To workaround this issue, we override `with_cppflags` for debase-ruby_core_source to still include `$CPPFLAGS`.
-  Debase::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
+  # To workaround this issue, we override `with_cppflags` for datadog-ruby_core_source to still include `$CPPFLAGS`.
+  Datadog::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
     super("#{newflags} #{$CPPFLAGS}", &block)
   end
 
-  Debase::RubyCoreSource
+  Datadog::RubyCoreSource
     .create_makefile_with_core(
       proc do
         headers_available =

ext/datadog_profiling_native_extension/native_extension_helpers.rb

@@ -9,7 +9,7 @@ module NativeExtensionHelpers
       # Can be set to force rubygems to fail gem installation when profiling extension could not be built
       ENV_FAIL_INSTALL_IF_MISSING_EXTENSION = "DD_PROFILING_FAIL_INSTALL_IF_MISSING_EXTENSION"
 
-      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on debase-ruby_core_source
+      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on datadog-ruby_core_source
       CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?("2.6", "2.7", "3.0.", "3.1.", "3.2.")
 
       def self.fail_install_if_missing_extension?

ext/datadog_profiling_native_extension/private_vm_api_access.c

@@ -13,7 +13,7 @@
   #include RUBY_MJIT_HEADER
 #else
   // The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  // the debase-ruby_core_source gem to get access to private VM headers.
+  // the datadog-ruby_core_source gem to get access to private VM headers.
 
   // We can't do anything about warnings in VM headers, so we just use this technique to suppress them.
   // See https://nelkinda.com/blog/suppress-warnings-in-gcc-and-clang/#d11e364 for details.

flake.nix

@@ -0,0 +1,57 @@
+{
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/master";
+
+    # cross-platform convenience
+    flake-utils.url = "github:numtide/flake-utils";
+
+    # backwards compatibility with nix-build and nix-shell
+    flake-compat.url = "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, flake-compat }:
+    # resolve for all platforms in turn
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        # packages for this system platform
+        pkgs = nixpkgs.legacyPackages.${system};
+
+        # control versions
+        ruby = pkgs.ruby_3_3;
+        llvm = pkgs.llvmPackages_16;
+        gcc = pkgs.gcc13;
+      in {
+        devShell = pkgs.llvm.stdenv.mkDerivation {
+          name = "devshell";
+
+          buildInputs = with pkgs; [
+            ruby
+            libyaml.dev
+
+            # TODO: some gems insist on using `gcc` on Linux, satisfy them for now:
+            # - json
+            # - protobuf
+            # - ruby-prof
+            gcc
+          ];
+
+          shellHook = ''
+            # get major.minor.0 ruby version
+            export RUBY_VERSION="$(ruby -e 'puts RUBY_VERSION.gsub(/\d+$/, "0")')"
+
+            # make gem install work in-project, compatibly with bundler
+            export GEM_HOME="$(pwd)/vendor/bundle/ruby/$RUBY_VERSION"
+
+            # make bundle work in-project
+            export BUNDLE_PATH="$(pwd)/vendor/bundle"
+
+            # enable calling gem scripts without bundle exec
+            export PATH="$GEM_HOME/bin:$PATH"
+
+            # enable implicitly resolving gems to bundled version
+            export RUBYGEMS_GEMDEPS="$(pwd)/Gemfile"
+          '';
+        };
+      }
+    );
+}