From ab102ded607b64df3ba71eb0c84a8a7d248f2ace Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Mon, 3 Feb 2025 15:41:19 -0700 Subject: [PATCH 1/6] Fixed generate_api workflow The workflow missed `sudo apt-get install libcurl4-openssl-dev` to install `curb` on ubuntu. Signed-off-by: Theo Truong --- .github/workflows/generate_api.yml | 1 + .github/workflows/main.yml | 3 +-- .github/workflows/test-unreleased.yml | 2 +- Gemfile | 37 +++++++++++++++------------ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/generate_api.yml b/.github/workflows/generate_api.yml index 5adece204..edcd6f48d 100644 --- a/.github/workflows/generate_api.yml +++ b/.github/workflows/generate_api.yml @@ -29,6 +29,7 @@ jobs: - name: Update bundler run: | sudo apt-get update + sudo apt-get install libcurl4-openssl-dev bundle install - name: Generate API diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5888ae8f4..364afafcb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,7 +48,7 @@ jobs: env: TEST_OPENSEARCH_SERVER: http://localhost:9250 PORT: 9250 - FARADAY_VERSION: '~> 1.0' + FARADAY_VERSION: '~> 1' strategy: fail-fast: false matrix: @@ -106,7 +106,6 @@ jobs: run: | sudo apt-get update sudo apt-get install libcurl4-openssl-dev - sudo apt-get install ca-certificates ruby -v bundle install - name: Test Client Security diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index 036bc2e49..476bdcc17 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -24,7 +24,7 @@ jobs: entry: - { ruby_version: '3.3', opensearch_ref: '1.x', jdk_version: '11' } - { ruby_version: '3.3', opensearch_ref: '2.x', jdk_version: '17' } - - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '17' } + - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '23' } steps: - uses: ruby/setup-ruby@v1 diff --git a/Gemfile b/Gemfile index 1edfa3201..7061cf664 100644 --- a/Gemfile +++ b/Gemfile @@ -28,19 +28,16 @@ source 'https://rubygems.org' gem 'opensearch-ruby', path: __dir__, require: false +# TODO: remove unnecessary dependencies gem 'ansi' gem 'bundler' gem 'cane' -gem 'faraday-httpclient' -gem 'faraday-net_http_persistent' gem 'hashie' -gem 'httpclient' gem 'jbuilder' gem 'jsonify' gem 'minitest', '~> 5' gem 'minitest-reporters', '~> 1' gem 'mocha', '~> 2' -gem 'net-http-persistent' gem 'pry' gem 'rake', '~> 13' gem 'rspec', '~> 3' @@ -50,19 +47,27 @@ gem 'rubocop-rspec' gem 'shoulda-context' gem 'simplecov', '~> 0.17', '< 0.18' gem 'test-unit', '~> 2' -gem 'typhoeus', '~> 1.4' gem 'webmock', '~> 2.0' gem 'yard' -gem 'curb' unless defined? JRUBY_VERSION -gem 'faraday-patron' unless defined? JRUBY_VERSION -gem 'patron' unless defined? JRUBY_VERSION +if defined? JRUBY_VERSION + gem 'manticore' + gem 'pry-nav' +else + gem 'curb' + gem 'require-prof' + gem 'ruby-prof' +end -gem 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius) -gem 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius) - -gem 'manticore' if defined? JRUBY_VERSION -gem 'pry-nav' if defined? JRUBY_VERSION - -gem 'faraday', ENV.fetch('FARADAY_VERSION', nil), require: false if ENV.key?('FARADAY_VERSION') -gem 'faraday-typhoeus' if !ENV.key?('FARADAY_VERSION') && Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') +if ENV.key?('FARADAY_VERSION') + gem 'faraday', ENV.fetch('FARADAY_VERSION'), require: false + gem 'httpclient' + gem 'net-http-persistent' + gem 'patron' unless defined? JRUBY_VERSION + gem 'typhoeus' +else + gem 'faraday-httpclient' + gem 'faraday-net_http_persistent' + gem 'faraday-patron' unless defined? JRUBY_VERSION + gem 'faraday-typhoeus' +end From 85c437a77d62e16023ca0218a7a699af87e64066 Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 6 Feb 2025 08:38:39 -0500 Subject: [PATCH 2/6] Use JDK version as part of the cache key. Signed-off-by: dblock --- .github/workflows/test-unreleased.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index 476bdcc17..970e80ccc 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -3,12 +3,12 @@ name: Integration with Unreleased OpenSearch on: push: branches: - - "main" + - 'main' paths-ignore: - '*.md' pull_request: branches: - - "main" + - 'main' paths-ignore: - '*.md' @@ -24,7 +24,7 @@ jobs: entry: - { ruby_version: '3.3', opensearch_ref: '1.x', jdk_version: '11' } - { ruby_version: '3.3', opensearch_ref: '2.x', jdk_version: '17' } - - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '23' } + - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '21' } steps: - uses: ruby/setup-ruby@v1 @@ -40,7 +40,7 @@ jobs: - name: Get OpenSearch branch top id: get-key working-directory: opensearch - run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT + run: echo key=`git log -1 --format='%H'`-${{ matrix.entry.jdk_version }} >> $GITHUB_OUTPUT - name: Restore cached build id: cache-restore @@ -60,7 +60,6 @@ jobs: if: steps.cache-restore.outputs.cache-hit != 'true' working-directory: opensearch run: ./gradlew :distribution:archives:linux-tar:assemble --stacktrace - - name: Save cached build if: steps.cache-restore.outputs.cache-hit != 'true' From 95ab9372104c8491a868af1a067ba9873716c05d Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 6 Feb 2025 08:40:54 -0500 Subject: [PATCH 3/6] Use typhoeus. Signed-off-by: dblock --- Gemfile | 2 +- spec/opensearch/transport/client_spec.rb | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index 7061cf664..7d8464312 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ if ENV.key?('FARADAY_VERSION') gem 'httpclient' gem 'net-http-persistent' gem 'patron' unless defined? JRUBY_VERSION - gem 'typhoeus' + gem 'typhoeus', '~> 1.4' else gem 'faraday-httpclient' gem 'faraday-net_http_persistent' diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index b380b1164..6c5a8b3bc 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1762,40 +1762,45 @@ end end - context 'when patron is used as an adapter', unless: jruby? do + context 'when typhoeus is used as an adapter', unless: jruby? do before do - require 'patron' + require 'typhoeus' end let(:options) do - { adapter: :patron } + { adapter: :typhoeus } end let(:adapter) do client.transport.connections.first.connection.builder.adapter end - it 'uses the patron connection handler' do - expect(adapter).to eq('Faraday::Adapter::Patron') + it 'uses the typhoeus connection handler' do + expect(adapter).to eq('Faraday::Adapter::Typhoeus') end it 'keeps connections open' do + puts 'Testing with _cat/health' + puts client.perform_request('GET', '_cat/health') + puts 'Performing GET _nodes/stats/http' response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] + puts 'Start reloading connections' client.transport.reload_connections! + puts 'Connections reloaded' response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) end end - context 'when typhoeus is used as an adapter', unless: jruby? do + context 'when patron is used as an adapter', unless: jruby? do before do - require 'typhoeus' + require 'patron' end let(:options) do - { adapter: :typhoeus } + { adapter: :patron } end let(:adapter) do @@ -1803,7 +1808,7 @@ end it 'uses the patron connection handler' do - expect(adapter).to eq('Faraday::Adapter::Typhoeus') + expect(adapter).to eq('Faraday::Adapter::Patron') end it 'keeps connections open' do From 77fb9c47d347525781150ff21d41a0867dae8599 Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 6 Feb 2025 08:46:31 -0500 Subject: [PATCH 4/6] Always set the required JDK. Signed-off-by: dblock --- .github/workflows/test-unreleased.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index 970e80ccc..e7140916b 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -51,7 +51,6 @@ jobs: - name: Setup Java JDK uses: actions/setup-java@v4 - if: steps.cache-restore.outputs.cache-hit != 'true' with: distribution: 'temurin' java-version: ${{ matrix.entry.jdk_version }} From 28b773accbeeeb930eaf900e1f186622c3128f1c Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 6 Feb 2025 09:42:09 -0500 Subject: [PATCH 5/6] Downgrade to Faraday 1.x. Signed-off-by: dblock --- .github/workflows/test-unreleased.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index e7140916b..63a826fad 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -15,6 +15,7 @@ on: jobs: test: env: + FARADAY_VERSION: '~> 1' TEST_OPENSEARCH_SERVER: http://localhost:9200 PORT: 9200 runs-on: ubuntu-latest From 240843690cbb54cba55eef83b92201b000ae20cd Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 6 Feb 2025 09:47:21 -0500 Subject: [PATCH 6/6] Undo debugging changes. Signed-off-by: dblock --- spec/opensearch/transport/client_spec.rb | 25 ++++++++++-------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index 6c5a8b3bc..a39f6921f 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1762,53 +1762,48 @@ end end - context 'when typhoeus is used as an adapter', unless: jruby? do + context 'when patron is used as an adapter', unless: jruby? do before do - require 'typhoeus' + require 'patron' end let(:options) do - { adapter: :typhoeus } + { adapter: :patron } end let(:adapter) do client.transport.connections.first.connection.builder.adapter end - it 'uses the typhoeus connection handler' do - expect(adapter).to eq('Faraday::Adapter::Typhoeus') + it 'uses the patron connection handler' do + expect(adapter).to eq('Faraday::Adapter::Patron') end it 'keeps connections open' do - puts 'Testing with _cat/health' - puts client.perform_request('GET', '_cat/health') - puts 'Performing GET _nodes/stats/http' response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] - puts 'Start reloading connections' client.transport.reload_connections! - puts 'Connections reloaded' response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) end end - context 'when patron is used as an adapter', unless: jruby? do + context 'when typhoeus is used as an adapter', unless: jruby? do before do - require 'patron' + require 'typhoeus' end let(:options) do - { adapter: :patron } + { adapter: :typhoeus } end let(:adapter) do client.transport.connections.first.connection.builder.adapter end - it 'uses the patron connection handler' do - expect(adapter).to eq('Faraday::Adapter::Patron') + it 'uses the typhoeus connection handler' do + expect(adapter).to eq('Faraday::Adapter::Typhoeus') end it 'keeps connections open' do