diff --git a/engines/data_export/README.md b/engines/data_export/README.md new file mode 100644 index 000000000..a0ee33843 --- /dev/null +++ b/engines/data_export/README.md @@ -0,0 +1,11 @@ +# DataExport +The data export engine supports exporting of data from RMT whenever a +client visits the RMT server to received updates or an initial registration +is performed. + +With this implementation the implementer can decide to forward all client +data or specific data to some external service for data analysis for example. + +## Usage +Place you implementation into /usr/share/rmt/engines/data_export/lib/data_export/handlers/ and implement the export_rmt_data function. The function takes no +arguments. Data to be exported is extracted from the RMT data base. diff --git a/engines/data_export/config/routes.rb b/engines/data_export/config/routes.rb new file mode 100644 index 000000000..31d295fb2 --- /dev/null +++ b/engines/data_export/config/routes.rb @@ -0,0 +1,2 @@ +DataExport::Engine.routes.draw do # rubocop:disable Lint/EmptyBlock +end diff --git a/engines/data_export/lib/data_export.rb b/engines/data_export/lib/data_export.rb new file mode 100644 index 000000000..45823d00e --- /dev/null +++ b/engines/data_export/lib/data_export.rb @@ -0,0 +1,28 @@ +$LOAD_PATH.push File.expand_path(__dir__, '..') + +module DataExport + class << self + # rubocop:disable ThreadSafety/ClassAndModuleAttributes + attr_accessor :handler + # rubocop:enable ThreadSafety/ClassAndModuleAttributes + end + + class Exception < RuntimeError; end +end + +module DataExport::Handlers +end + +require 'data_export/engine' +require 'data_export/handler_base' + +handlers = Dir.glob(File.join(__dir__, 'data_export/handlers/*.rb')) + +raise 'Too many data export handlers found' if handlers.size > 1 + +# rubocop:disable Lint:UnreachableLoop +handlers.each do |f| + require_relative f + break +end +# rubocop:enable Lint:UnreachableLoop diff --git a/engines/data_export/lib/data_export/engine.rb b/engines/data_export/lib/data_export/engine.rb new file mode 100644 index 000000000..4137caf59 --- /dev/null +++ b/engines/data_export/lib/data_export/engine.rb @@ -0,0 +1,53 @@ +module DataExport + class Engine < ::Rails::Engine + isolate_namespace DataExport + config.after_initialize do + # replaces RMT registration for SCC registration + Api::Connect::V3::Subscriptions::SystemsController.class_eval do + after_action :export_rmt_data, only: %i[announce_system], if: -> { response.successful? } + + def export_rmt_data + # no need to check if system is nil + # as the response is successful + return if @system.byos? + + data_export_handler = DataExport.handler.new( + @system, + request, + params, + logger + ) + data_export_handler.export_rmt_data + logger.info "System #{@system.login} info updated by data export handler after announcing the system" + rescue StandardError => e + logger.error('Unexpected data export error has occurred:') + logger.error(e.message) + logger.error("System login: #{@system.login}, IP: #{request.remote_ip}") + logger.error('Backtrace:') + logger.error(e.backtrace) + end + end + + Api::Connect::V3::Systems::SystemsController.class_eval do + after_action :export_rmt_data, only: %i[update], if: -> { response.successful? } + + def export_rmt_data + data_export_handler = DataExport.handler.new( + @system, + request, + params, + logger + ) + data_export_handler.export_rmt_data + logger.info "System #{@system.login} info updated by data export handler after updating the system" + rescue StandardError => e + logger.error('Unexpected data export error has occurred:') + logger.error(e.message) + logger.error("System login: #{@system.login}, IP: #{request.remote_ip}") + logger.error('Backtrace:') + logger.error(e.backtrace) + end + end + end + end +end diff --git a/engines/data_export/lib/data_export/handler_base.rb b/engines/data_export/lib/data_export/handler_base.rb new file mode 100644 index 000000000..7c1afb43c --- /dev/null +++ b/engines/data_export/lib/data_export/handler_base.rb @@ -0,0 +1,12 @@ +class DataExport::HandlerBase + def self.inherited(child_class) # rubocop:disable Lint/MissingSuper + DataExport.handler = child_class + end + + def initialize(system, request, params, logger) + @system = system + @request = request + @params = params + @logger = logger + end +end diff --git a/engines/data_export/lib/data_export/handlers/example.rb b/engines/data_export/lib/data_export/handlers/example.rb new file mode 100644 index 000000000..c9f00db6a --- /dev/null +++ b/engines/data_export/lib/data_export/handlers/example.rb @@ -0,0 +1,5 @@ +class DataExport::Handlers::Example < DataExport::HandlerBase + def export_rmt_data + true + end +end diff --git a/engines/data_export/lib/tasks/data_export_tasks.rake b/engines/data_export/lib/tasks/data_export_tasks.rake new file mode 100644 index 000000000..2a85b8676 --- /dev/null +++ b/engines/data_export/lib/tasks/data_export_tasks.rake @@ -0,0 +1,4 @@ +# desc "Explaining what the task does" +# task :data_export do +# # Task goes here +# end diff --git a/engines/data_export/spec/requests/api/connect/v3/subscriptions/systems_controller_spec.rb b/engines/data_export/spec/requests/api/connect/v3/subscriptions/systems_controller_spec.rb new file mode 100644 index 000000000..a73dd41a5 --- /dev/null +++ b/engines/data_export/spec/requests/api/connect/v3/subscriptions/systems_controller_spec.rb @@ -0,0 +1,80 @@ +require 'rails_helper' + +# rubocop:disable RSpec/NestedGroups +describe Api::Connect::V3::Subscriptions::SystemsController, type: :request do + describe '#announce_system' do + let(:instance_data) { '' } + + context 'using SCC generated credentials (BYOS mode)' do + let(:scc_register_system_url) { 'https://scc.suse.com/connect/subscriptions/systems' } + let(:scc_register_response) do + { + id: 5684096, + login: 'SCC_foo', + password: '1234', + last_seen_at: '2021-10-24T09:48:52.658Z' + }.to_json + end + let(:params) do + { + hostname: 'test', + proxy_byos_mode: :payg, + instance_data: instance_data, + hwinfo: + { + hostname: 'test', + cpus: '1', + sockets: '1', + hypervisor: 'Xen', + arch: 'x86_64', + uuid: 'ec235f7d-b435-e27d-86c6-c8fef3180a01', + cloud_provider: 'super_cloud' + } + } + end + + context 'valid credentials' do + let(:plugin_double) { instance_double('DataExport::Handlers::Example') } + + before do + allow(DataExport::Handlers::Example).to receive(:new).and_return(plugin_double) + allow(plugin_double).to receive(:export_rmt_data) + stub_request(:post, scc_register_system_url) + .to_return( + status: 201, + body: scc_register_response.to_s, + headers: {} + ) + end + + it 'saves the data' do + expect(plugin_double).to receive(:export_rmt_data) + post '/connect/subscriptions/systems', params: params, headers: { HTTP_AUTHORIZATION: 'Token token=' } + end + + context 'export fails' do + let(:logger) { instance_double('RMT::Logger').as_null_object } + + before do + allow(DataExport::Handlers::Example).to receive(:new).and_return(plugin_double) + allow(plugin_double).to receive(:export_rmt_data).and_raise('foo') + allow(Rails.logger).to receive(:error) + stub_request(:post, scc_register_system_url) + .to_return( + status: 201, + body: scc_register_response.to_s, + headers: {} + ) + end + + it 'does not save the data and log error' do + expect(plugin_double).to receive(:export_rmt_data) + expect(Rails.logger).to receive(:error) + post '/connect/subscriptions/systems', params: params, headers: { HTTP_AUTHORIZATION: 'Token token=' } + end + end + end + end + end +end +# rubocop:enable RSpec/NestedGroups diff --git a/engines/data_export/spec/requests/api/connect/v3/systems/systems_controller_spec.rb b/engines/data_export/spec/requests/api/connect/v3/systems/systems_controller_spec.rb new file mode 100644 index 000000000..27e72b5c9 --- /dev/null +++ b/engines/data_export/spec/requests/api/connect/v3/systems/systems_controller_spec.rb @@ -0,0 +1,55 @@ +require 'rails_helper' + +RSpec.describe Api::Connect::V3::Systems::SystemsController do + include_context 'auth header', :system, :login, :password + include_context 'version header', 3 + include_context 'user-agent header' + include_context 'zypp user-agent header' + + let(:system) { FactoryBot.create(:system, hostname: 'initial') } + let(:url) { '/connect/systems' } + let(:headers) { auth_header.merge(version_header) } + let(:hwinfo) do + { + cpus: 16, + sockets: 1, + arch: 'x86_64', + hypervisor: 'XEN', + uuid: 'f46906c5-d87d-4e4c-894b-851e80376003', + cloud_provider: 'testcloud' + } + end + let(:payload) { { hostname: 'test', hwinfo: hwinfo } } + let(:system_uptime) { system.system_uptimes.first } + let(:plugin_double) { instance_double('DataExport::Handlers::Example') } + + describe '#update' do + subject(:update_action) { put url, params: payload, headers: headers } + + context 'when update success' do + before { allow(DataExport::Handlers::Example).to receive(:new).and_return(plugin_double) } + + context 'when data export success' do + before { allow(plugin_double).to receive(:export_rmt_data) } + + it do + expect(plugin_double).to receive(:export_rmt_data) + expect { update_action }.to change { system.reload.hostname }.from('initial').to(payload[:hostname]) + end + end + + context 'when data export fails' do + before do + allow(plugin_double).to receive(:export_rmt_data).and_raise('foo') + allow(Rails.logger).to receive(:error) + end + + it do + expect(plugin_double).to receive(:export_rmt_data) + expect(Rails.logger).to receive(:error) + update_action + end + end + end + end +end diff --git a/engines/scc_proxy/lib/scc_proxy/engine.rb b/engines/scc_proxy/lib/scc_proxy/engine.rb index 1e74bba4a..e728fa98f 100644 --- a/engines/scc_proxy/lib/scc_proxy/engine.rb +++ b/engines/scc_proxy/lib/scc_proxy/engine.rb @@ -281,7 +281,7 @@ def announce_system system_information: system_information, proxy_byos_mode: :payg, instance_data: instance_data - ) + ) else request.request_parameters['proxy_byos_mode'] = 'byos' response = SccProxy.announce_system_scc(auth_header, request.request_parameters) diff --git a/engines/zypper_auth/lib/zypper_auth/engine.rb b/engines/zypper_auth/lib/zypper_auth/engine.rb index fcf21ce31..41817b41f 100644 --- a/engines/zypper_auth/lib/zypper_auth/engine.rb +++ b/engines/zypper_auth/lib/zypper_auth/engine.rb @@ -139,7 +139,10 @@ def path_allowed?(headers) return true if @system.byos? - ZypperAuth.verify_instance(request, logger, @system) + instance_verified = ZypperAuth.verify_instance(request, logger, @system) + DataExport.handler.new(@system, request, params, logger).export_rmt_data if instance_verified + + instance_verified end end end diff --git a/engines/zypper_auth/spec/requests/strict_authentication/authentication_controller_spec.rb b/engines/zypper_auth/spec/requests/strict_authentication/authentication_controller_spec.rb index 5e5026efb..d8bac081d 100644 --- a/engines/zypper_auth/spec/requests/strict_authentication/authentication_controller_spec.rb +++ b/engines/zypper_auth/spec/requests/strict_authentication/authentication_controller_spec.rb @@ -16,14 +16,17 @@ let(:requested_uri) { '/repo' + system.repositories.first[:local_path] + '/repodata/repomd.xml' } let(:paid_requested_uri) { '/repo' + system.products.where(free: false).first.repositories.first[:local_path] + '/repodata/repomd.xml' } + let(:data_export_double) { instance_double('DataExport::Handlers::Example') } context 'without instance_data headers' do let(:headers) { auth_header.merge({ 'X-Original-URI': requested_uri }) } before do + allow(DataExport::Handlers::Example).to receive(:new).and_return(data_export_double) allow(File).to receive(:directory?) allow(Dir).to receive(:mkdir) allow(FileUtils).to receive(:touch) + expect(data_export_double).not_to receive(:export_rmt_data) get '/api/auth/check', headers: headers end @@ -39,6 +42,7 @@ allow(File).to receive(:directory?) allow(Dir).to receive(:mkdir) allow(FileUtils).to receive(:touch) + expect(data_export_double).not_to receive(:export_rmt_data) get '/api/auth/check', headers: headers end @@ -269,14 +273,17 @@ context 'with instance_data headers and instance data is valid' do let(:headers) { auth_header.merge({ 'X-Original-URI': requested_uri, 'X-Instance-Data': 'test' }) } + let(:data_export_double) { instance_double('DataExport::Handlers::Example') } before do Rails.cache.clear expect_any_instance_of(InstanceVerification::Providers::Example).to receive(:instance_valid?).and_return(true) allow(InstanceVerification).to receive(:update_cache) + allow(DataExport::Handlers::Example).to receive(:new).and_return(data_export_double) allow(File).to receive(:directory?) allow(Dir).to receive(:mkdir) allow(FileUtils).to receive(:touch) + expect(data_export_double).to receive(:export_rmt_data) get '/api/auth/check', headers: headers end @@ -491,8 +498,12 @@ end context 'the path to check is free' do + let(:data_export_double) { instance_double('DataExport::Handlers::Example') } + before do + allow(DataExport::Handlers::Example).to receive(:new).and_return(data_export_double) expect_any_instance_of(InstanceVerification::Providers::Example).to receive(:instance_valid?).and_return(true) + expect(data_export_double).to receive(:export_rmt_data) get '/api/auth/check', headers: headers end diff --git a/package/obs/rmt-server.changes b/package/obs/rmt-server.changes index fc72bc88f..bb25018a3 100644 --- a/package/obs/rmt-server.changes +++ b/package/obs/rmt-server.changes @@ -17,6 +17,7 @@ Fri Jan 03 10:44:00 UTC 2025 - Luís Caparroz * rmt-server-pubcloud: * Update Micro check due to Micro 6.0 and 6.1 identifier to keep bsc#1230419 in place * Update Zypper path allowing check to handle paid extensions (i.e. LTSS) (bsc#1230157) + * Add data export engine ------------------------------------------------------------------- Mon Dec 23 08:03:56 UTC 2024 - Parag Jain