Merge remote-tracking branch 'origin/master' into HEAD

* origin/master: (51 commits)
  DEBUG-2334 duplicate mutable values when serializing for dynamic inst… (DataDog#4009)
  DEBUG-2334 enforce probe type validity (DataDog#4013)
  [🤖] Update Latest Dependency: https://github.com/DataDog/dd-trace-rb/actions/runs/11421728295
  Fix the argument to the telemetry forwarder command
  [🤖] Lock Dependency: https://github.com/DataDog/dd-trace-rb/actions/runs/11460992004
  Add datadog gem to gemspec and remove from Gemfile
  Replace debase with datadog, and comment out gemspec tests
  Add datadog gem to Gemfile
  Remove debase gem from gemspec
  Use nix develop
  Use Ubuntu 24.04 by Arm Limited
  Fix vendored dependency case
  revert system-tests branch to main
  Changed RuleSampler initialization with ASM Standalone to Tracing::Component.build_sampler
  Rename AppSec::Event.add_tags to AppSec::Event.tag_and_keep! and move trace.keep in it
  Move appsec_standalone_reject? to AppSec namespace
  Replaced set_tag by set_metric for _dd.appsec.enabled and _dd.apm.enabled metrics
  Add correct sig to Datadog::AppSec::Event.add_tags and add_distributed_tags
  Update Unreleased Changelog
  Fix typo in AppSec::Event.add_tags spec
  ...

.github/workflows/nix.yml

@@ -0,0 +1,55 @@
+name: Test Nix
+
+on:
+  push:
+    branches:
+      - "**"
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - os: darwin
+            cpu: x86_64
+            base: macos-13 # always x86_64-darwin
+          - os: darwin
+            cpu: arm64
+            base: macos-14 # always arm64-darwin
+          - os: linux
+            cpu: x86_64
+            base: ubuntu-24.04 # always x86_64-linux-gnu
+          - os: linux
+            cpu: aarch64
+            base: arm-4core-linux-ubuntu24.04 # always aarch64-linux-gnu
+        nix:
+          - 24.05
+
+    name: Test Nix (${{ matrix.platform.cpu }}-${{ matrix.platform.os }}, ${{ matrix.nix }})
+    runs-on: ${{ matrix.platform.base }}
+
+    permissions:
+      contents: read
+      id-token: write
+
+    env:
+      SKIP_SIMPLECOV: 1
+      DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
+      DD_REMOTE_CONFIGURATION_ENABLED: false
+
+    steps:
+      - name: Check CPU arch
+        run: |
+          test "$(uname -m)" = "${{ matrix.platform.cpu }}"
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Print ruby version
+        run: |
+          nix develop --command which ruby
+          nix develop --command ruby --version
+      - name: Bundle install
+        run: nix develop --command bundle install
+      - name: Run spec:main
+        run: nix develop --command bundle exec rake spec:main

.github/workflows/system-tests.yml

@@ -12,8 +12,8 @@ env:
   REGISTRY: ghcr.io
   REPO: ghcr.io/datadog/dd-trace-rb
   ST_REF: main
-  FORCE_TESTS: -F tests/appsec/waf/test_addresses.py::Test_GraphQL -F tests/appsec/test_blocking_addresses.py::Test_BlockingGraphqlResolvers
-  FORCE_TESTS_SCENARIO: GRAPHQL_APPSEC
+  FORCE_TESTS: -F tests/appsec/test_asm_standalone.py
+  FORCE_TESTS_SCENARIO: APPSEC_STANDALONE
 
 jobs:
   build-harness:
@@ -199,6 +199,7 @@ jobs:
           - APPSEC_DISABLED
           - APPSEC_BLOCKING_FULL_DENYLIST
           - APPSEC_REQUEST_BLOCKING
+          - APPSEC_STANDALONE
         include:
           - library: ruby
             app: rack

.gitignore

@@ -38,7 +38,8 @@ ext/**/skipped_reason.txt
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
 # Ignore local variables
-.envrc
+/.envrc
+/.direnv
 
 # lock files
 Gemfile.lock

.gitlab/install_datadog_deps.rb

@@ -70,7 +70,7 @@
 }
 
 [
-  'debase-ruby_core_source',
+  'datadog-ruby_core_source',
   'ffi',
   'libddwaf',
   'msgpack',

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+* AppSec: Add Experimental Standalone AppSec Threats billing ([#3965][])
+
 ## [2.4.0] - 2024-10-11
 
 ### Added

datadog.gemspec

@@ -64,7 +64,7 @@ Gem::Specification.new do |spec|
   # Used by the profiler native extension to support Ruby < 2.6 and > 3.2
   #
   # We decided to pin it at the latest available version and will manually bump the dependency as needed.
-  spec.add_dependency 'debase-ruby_core_source', '= 3.3.1'
+  spec.add_dependency 'datadog-ruby_core_source', '= 3.3.6'
 
   # Used by appsec
   spec.add_dependency 'libddwaf', '~> 1.14.0.0.0'

default.nix

@@ -0,0 +1,11 @@
+# flake-compat shim for usage without flakes
+(import
+  (
+    let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
+    fetchTarball {
+      url = lock.nodes.flake-compat.locked.url or "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+      sha256 = lock.nodes.flake-compat.locked.narHash;
+    }
+  )
+  { src = ./.; }
+).defaultNix

ext/datadog_profiling_native_extension/NativeExtensionDesign.md

@@ -68,7 +68,7 @@ internal types, structures and functions).
 Because these private header files are not included in regular Ruby installations, we have two different workarounds:
 
 1. for Ruby versions 2.6 to 3.2 we make use use the Ruby private MJIT header
-2. for Ruby versions < 2.6 and > 3.2 we make use of the `debase-ruby_core_source` gem
+2. for Ruby versions < 2.6 and > 3.2 we make use of the `datadog-ruby_core_source` gem
 
 Functions which make use of these headers are defined in the <private_vm_api_acccess.c> file.
 
@@ -91,9 +91,9 @@ version. e.g. `rb_mjit_min_header-2.7.4.h`.
 
 This header was removed in Ruby 3.3.
 
-### Approach 2: Using the `debase-ruby_core_source` gem
+### Approach 2: Using the `datadog-ruby_core_source` gem
 
-The [`debase-ruby_core_source`](https://github.com/ruby-debug/debase-ruby_core_source) contains almost no code;
+The [`datadog-ruby_core_source`](https://github.com/DataDog/datadog-ruby_core_source) contains almost no code;
 instead, it just contains per-Ruby-version folders with the private VM headers (`.h`) files for that version.
 
 Thus, even though a regular Ruby installation does not include these files, we can access the copy inside this gem.

ext/datadog_profiling_native_extension/extconf.rb

@@ -256,21 +256,21 @@ def skip_building_extension!(reason)
   create_makefile EXTENSION_NAME
 else
   # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  # the debase-ruby_core_source gem to get access to private VM headers.
+  # the datadog-ruby_core_source gem to get access to private VM headers.
   # This gem ships source code copies of these VM headers for the different Ruby VM versions;
-  # see https://github.com/ruby-debug/debase-ruby_core_source for details
+  # see https://github.com/DataDog/datadog-ruby_core_source for details
 
   create_header
 
-  require "debase/ruby_core_source"
+  require "datadog/ruby_core_source"
   dir_config("ruby") # allow user to pass in non-standard core include directory
 
   # This is a workaround for a weird issue...
   #
-  # The mkmf tool defines a `with_cppflags` helper that debase-ruby_core_source uses. This helper temporarily
+  # The mkmf tool defines a `with_cppflags` helper that datadog-ruby_core_source uses. This helper temporarily
   # replaces `$CPPFLAGS` (aka the C pre-processor [not c++!] flags) with a different set when doing something.
   #
-  # The debase-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
+  # The datadog-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
   # path. But because `with_cppflags` replaces `$CPPFLAGS`, well, the default `$CPPFLAGS` are not included in the
   # makefile.
   #
@@ -281,12 +281,12 @@ def skip_building_extension!(reason)
   # `VM_CHECK_MODE=1` when building Ruby will trigger this issue (because somethings in structures the profiler reads
   # are ifdef'd out using this setting).
   #
-  # To workaround this issue, we override `with_cppflags` for debase-ruby_core_source to still include `$CPPFLAGS`.
-  Debase::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
+  # To workaround this issue, we override `with_cppflags` for datadog-ruby_core_source to still include `$CPPFLAGS`.
+  Datadog::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
     super("#{newflags} #{$CPPFLAGS}", &block)
   end
 
-  Debase::RubyCoreSource
+  Datadog::RubyCoreSource
     .create_makefile_with_core(
       proc do
         headers_available =

ext/datadog_profiling_native_extension/native_extension_helpers.rb

@@ -9,7 +9,7 @@ module NativeExtensionHelpers
       # Can be set to force rubygems to fail gem installation when profiling extension could not be built
       ENV_FAIL_INSTALL_IF_MISSING_EXTENSION = "DD_PROFILING_FAIL_INSTALL_IF_MISSING_EXTENSION"
 
-      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on debase-ruby_core_source
+      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on datadog-ruby_core_source
       CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?("2.6", "2.7", "3.0.", "3.1.", "3.2.")
 
       def self.fail_install_if_missing_extension?

ext/datadog_profiling_native_extension/private_vm_api_access.c

@@ -13,7 +13,7 @@
   #include RUBY_MJIT_HEADER
 #else
   // The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  // the debase-ruby_core_source gem to get access to private VM headers.
+  // the datadog-ruby_core_source gem to get access to private VM headers.
 
   // We can't do anything about warnings in VM headers, so we just use this technique to suppress them.
   // See https://nelkinda.com/blog/suppress-warnings-in-gcc-and-clang/#d11e364 for details.

flake.nix

@@ -0,0 +1,57 @@
+{
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/master";
+
+    # cross-platform convenience
+    flake-utils.url = "github:numtide/flake-utils";
+
+    # backwards compatibility with nix-build and nix-shell
+    flake-compat.url = "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, flake-compat }:
+    # resolve for all platforms in turn
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        # packages for this system platform
+        pkgs = nixpkgs.legacyPackages.${system};
+
+        # control versions
+        ruby = pkgs.ruby_3_3;
+        llvm = pkgs.llvmPackages_16;
+        gcc = pkgs.gcc13;
+      in {
+        devShell = pkgs.llvm.stdenv.mkDerivation {
+          name = "devshell";
+
+          buildInputs = with pkgs; [
+            ruby
+            libyaml.dev
+
+            # TODO: some gems insist on using `gcc` on Linux, satisfy them for now:
+            # - json
+            # - protobuf
+            # - ruby-prof
+            gcc
+          ];
+
+          shellHook = ''
+            # get major.minor.0 ruby version
+            export RUBY_VERSION="$(ruby -e 'puts RUBY_VERSION.gsub(/\d+$/, "0")')"
+
+            # make gem install work in-project, compatibly with bundler
+            export GEM_HOME="$(pwd)/vendor/bundle/ruby/$RUBY_VERSION"
+
+            # make bundle work in-project
+            export BUNDLE_PATH="$(pwd)/vendor/bundle"
+
+            # enable calling gem scripts without bundle exec
+            export PATH="$GEM_HOME/bin:$PATH"
+
+            # enable implicitly resolving gems to bundled version
+            export RUBYGEMS_GEMDEPS="$(pwd)/Gemfile"
+          '';
+        };
+      }
+    );
+}