From 4779ac1d3bc98d1afddbccb2096fdd75b62a7f31 Mon Sep 17 00:00:00 2001 From: Davis Plumlee <56367316+dplumlee@users.noreply.github.com> Date: Sun, 23 Feb 2025 20:18:02 -0500 Subject: [PATCH] [Security Solution] Adds prebuilt rule import/export integration tests (#206893) ## Summary Adds integration tests in accordance to https://github.com/elastic/kibana/pull/204889 Adds on to the existing tests we have for rule import and export to include tests related to the prebuilt rule customization epic and the new functionality that will be shipping. All these tests are running behind the `prebuiltRulesCustomizationEnabled` feature flag. ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [x] ESS x100: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7921 - [x] Serverless x100: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7922 --------- Co-authored-by: Elastic Machine Co-authored-by: Georgii Gorbachev (cherry picked from commit 3e4ed6ebd58c77f555e2eb1287f70ad41ca73666) --- .../prebuilt_rules/prebuilt_rule_import.md | 59 ++++-- .../customization_enabled/import_rules.ts | 197 ++++++++++++++++-- .../cypress/tasks/alerts_detection_rules.ts | 1 + 3 files changed, 219 insertions(+), 38 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md index 68063421ce992..4896c54d7dd26 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md +++ b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md @@ -27,20 +27,20 @@ https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one - [Non-functional requirements](#non-functional-requirements) - [Scenarios](#scenarios) - [Core Functionality](#core-functionality) - - [Scenario: Importing an unmodified prebuilt rule with a matching rule\_id and version](#scenario-importing-an-unmodified-prebuilt-rule-with-a-matching-rule_id-and-version) - - [Scenario: Importing a customized prebuilt rule with a matching rule\_id and version](#scenario-importing-a-customized-prebuilt-rule-with-a-matching-rule_id-and-version) - - [Scenario: Importing a custom rule with a matching rule\_id and version](#scenario-importing-a-custom-rule-with-a-matching-rule_id-and-version) - - [Scenario: Importing a prebuilt rule with a matching rule\_id but no matching version](#scenario-importing-a-prebuilt-rule-with-a-matching-rule_id-but-no-matching-version) - - [Scenario: Importing a prebuilt rule with a non-existent rule\_id](#scenario-importing-a-prebuilt-rule-with-a-non-existent-rule_id) - - [Scenario: Importing a prebuilt rule without a rule\_id field](#scenario-importing-a-prebuilt-rule-without-a-rule_id-field) - - [Scenario: Importing a prebuilt rule with a matching rule\_id but missing a version field](#scenario-importing-a-prebuilt-rule-with-a-matching-rule_id-but-missing-a-version-field) + - [Scenario: Importing an unmodified prebuilt rule with a matching rule_id and version](#scenario-importing-an-unmodified-prebuilt-rule-with-a-matching-rule_id-and-version) + - [Scenario: Importing a customized prebuilt rule with a matching rule_id and version](#scenario-importing-a-customized-prebuilt-rule-with-a-matching-rule_id-and-version) + - [Scenario: Importing a custom rule with a matching rule_id and version](#scenario-importing-a-custom-rule-with-a-matching-rule_id-and-version) + - [Scenario: Importing a prebuilt rule with a matching rule_id but no matching version](#scenario-importing-a-prebuilt-rule-with-a-matching-rule_id-but-no-matching-version) + - [Scenario: Importing a prebuilt rule with a non-existent rule_id](#scenario-importing-a-prebuilt-rule-with-a-non-existent-rule_id) + - [Scenario: Importing a prebuilt rule without a rule_id field](#scenario-importing-a-prebuilt-rule-without-a-rule_id-field) + - [Scenario: Importing a prebuilt rule with a matching rule_id but missing a version field](#scenario-importing-a-prebuilt-rule-with-a-matching-rule_id-but-missing-a-version-field) - [Scenario: Importing an existing custom rule missing a version field](#scenario-importing-an-existing-custom-rule-missing-a-version-field) - [Scenario: Importing a new custom rule missing a version field](#scenario-importing-a-new-custom-rule-missing-a-version-field) - [Scenario: Importing a rule with overwrite flag set to true](#scenario-importing-a-rule-with-overwrite-flag-set-to-true) - [Scenario: Importing a rule with overwrite flag set to false](#scenario-importing-a-rule-with-overwrite-flag-set-to-false) - [Scenario: Importing both custom and prebuilt rules](#scenario-importing-both-custom-and-prebuilt-rules) - [Scenario: Importing prebuilt rules when the rules package is not installed](#scenario-importing-prebuilt-rules-when-the-rules-package-is-not-installed) - - [Scenario: User imports a custom rule before a prebuilt rule asset is created with the same rule\_id](#scenario-user-imports-a-custom-rule-before-a-prebuilt-rule-asset-is-created-with-the-same-rule_id) + - [Scenario: User imports a custom rule before a prebuilt rule asset is created with the same rule_id](#scenario-user-imports-a-custom-rule-before-a-prebuilt-rule-asset-is-created-with-the-same-rule_id) ## Useful information @@ -83,8 +83,8 @@ https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one **Automation**: 1 cypress test and 1 integration test. ```Gherkin -Given the import payload contains a prebuilt rule with a matching rule_id and version, identical to the published rule -When the user imports the rule +Given the import payload contains an unmodified prebuilt rule +And its rule_id and version match a rule asset from the installed package Then the rule should be created or updated And the ruleSource type should be "external" And isCustomized should be false @@ -95,17 +95,26 @@ And isCustomized should be false **Automation**: 1 cypress test and 1 integration test. ```Gherkin -Given the import payload contains a prebuilt rule with a matching rule_id and version, modified from the published version -And the overwrite flag is set to true +Given the import payload contains a modified prebuilt rule +And its rule_id and version match a rule asset from the installed package When the user imports the rule Then the rule should be created or updated And the ruleSource type should be "external" And isCustomized should be true +``` + +#### Scenario: Importing a custom rule with a matching prebuilt rule_id and version -CASE: Should work with older, newer, or identical version numbers +**Automation**: 1 cypress test and 1 integration test. + +```Gherkin +Given the import payload contains a custom rule with a matching rule_id and version +When the user imports the rule +Then the rule should be created or updated +And the ruleSource type should be "external" ``` -#### Scenario: Importing a custom rule with a matching rule_id and version +#### Scenario: Importing a custom rule with a matching custom rule_id and version **Automation**: 1 cypress test and 1 integration test. @@ -113,7 +122,7 @@ CASE: Should work with older, newer, or identical version numbers Given the import payload contains a custom rule with a matching rule_id and version And the overwrite flag is set to true When the user imports the rule -Then the rule should be updated +Then the rule should be created or updated And the ruleSource type should be "internal" ``` @@ -122,10 +131,11 @@ And the ruleSource type should be "internal" **Automation**: 1 integration test. ```Gherkin -Given the import payload contains a prebuilt rule with a matching rule_id but no matching version -And the overwrite flag is set to true +Given the import payload contains a prebuilt rule +And its rule_id matches a rule asset from the installed package +And the version does not match the rule asset's version When the user imports the rule -Then the rule should be created +Then the rule should be created or updated And the ruleSource type should be "external" And isCustomized should be true ``` @@ -135,7 +145,8 @@ And isCustomized should be true **Automation**: 1 integration test. ```Gherkin -Given the import payload contains a prebuilt rule with a non-existent rule_id +Given the import payload contains a prebuilt rule +And its rule_id does NOT match a rule asset from the installed package When the user imports the rule Then the rule should be created And the ruleSource type should be "internal" @@ -190,11 +201,12 @@ And the "version" field should be set to 1 **Automation**: 1 integration test. ```Gherkin -Given the import payload contains a rule with an existing rule_id +Given the import payload contains a rule +And its rule_id matches a rule_id of one of the installed rules And the overwrite flag is set to true When the user imports the rule Then the rule should be overwritten -And the ruleSource type should be calculated based on the rule_id and version +And the ruleSource should be based on rule_id and version ``` #### Scenario: Importing a rule with overwrite flag set to false @@ -202,7 +214,8 @@ And the ruleSource type should be calculated based on the rule_id and version **Automation**: 1 integration test. ```Gherkin -Given the import payload contains a rule with an existing rule_id +Given the import payload contains a rule +And its rule_id matches a rule_id of one of the installed rules And the overwrite flag is set to false When the user imports the rule Then the import should be rejected with a message "rule_id already exists" @@ -230,7 +243,7 @@ And prebuilt rules missing versions should be rejected Given the import payload contains prebuilt rules And no rules package has been installed locally When the user imports the rule -Then all rules should be created or updated as custom rules +Then the latest prebuilt rules package should get installed automatically ``` #### Scenario: User imports a custom rule before a prebuilt rule asset is created with the same rule_id diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/import_rules.ts index 934ee6460a5e2..009a88239df31 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/import_rules.ts @@ -11,10 +11,14 @@ import { SAMPLE_PREBUILT_RULES_WITH_HISTORICAL_VERSIONS, combineArrayToNdJson, createHistoricalPrebuiltRuleAssetSavedObjects, + createRuleAssetSavedObject, deleteAllPrebuiltRuleAssets, + deletePrebuiltRulesFleetPackage, fetchRule, getCustomQueryRuleParams, getInstalledRules, + getPrebuiltRulesAndTimelinesStatus, + installPrebuiltRules, } from '../../../../utils'; import { deleteAllRules } from '../../../../../../../common/utils/security_solution'; import { FtrProviderContext } from '../../../../../../ftr_provider_context'; @@ -24,12 +28,15 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const log = getService('log'); const securitySolutionApi = getService('securitySolutionApi'); + const retryService = getService('retry'); - const importRules = async (rules: unknown[]) => { + const importRules = async (rules: unknown[], overwrite?: boolean) => { const buffer = Buffer.from(combineArrayToNdJson(rules)); return securitySolutionApi - .importRules({ query: {} }) + .importRules({ + query: { overwrite }, + }) .attach('file', buffer, 'rules.ndjson') .expect('Content-Type', 'application/json; charset=utf-8') .expect(200); @@ -59,8 +66,38 @@ export default ({ getService }: FtrProviderContext): void => { }); describe('calculation of rule customization fields', () => { - it('defaults a versionless custom rule to "version: 1"', async () => { - const rule = getCustomQueryRuleParams({ rule_id: 'custom-rule', version: undefined }); + it('imports a rule with overwrite flag set to true', async () => { + await installPrebuiltRules(es, supertest); + const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: 1 }); + const { body } = await importRules([rule], true); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + }); + + it('rejects a rule with an existing rule_id when overwrite flag set to false', async () => { + await installPrebuiltRules(es, supertest); + const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: 1 }); + const { body } = await importRules([rule]); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { + message: `rule_id: \"rule-1\" already exists`, + status_code: 409, + }, + }); + }); + + it('imports a custom rule with a matching prebuilt rule_id and version', async () => { + const rule = getCustomQueryRuleParams({ + rule_id: prebuiltRules[0].rule_id, + version: prebuiltRules[0].version, + }); const { body } = await importRules([rule]); expect(body).toMatchObject({ @@ -70,6 +107,54 @@ export default ({ getService }: FtrProviderContext): void => { errors: [], }); + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); + expect(importedRule).toMatchObject({ + rule_id: rule.rule_id, + version: 1, + rule_source: { type: 'external' }, + immutable: true, + }); + }); + + it('imports a custom rule with a matching custom rule_id and version', async () => { + const customRuleId = 'custom-rule-id'; + await securitySolutionApi + .createRule({ body: getCustomQueryRuleParams({ rule_id: customRuleId, version: 1 }) }) + .expect(200); + + const rule = getCustomQueryRuleParams({ + rule_id: customRuleId, + version: 1, + }); + const { body } = await importRules([rule], true); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: customRuleId }); + expect(importedRule).toMatchObject({ + rule_id: customRuleId, + version: 1, + rule_source: { type: 'internal' }, + immutable: false, + }); + }); + + it('imports a new custom rule missing a version field', async () => { + const rule = getCustomQueryRuleParams({ rule_id: 'custom-rule', version: undefined }); + const { body } = await importRules([rule], true); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); expect(importedRule).toMatchObject({ rule_id: rule.rule_id, @@ -99,8 +184,54 @@ export default ({ getService }: FtrProviderContext): void => { }); }); + it('imports an existing custom rule missing a version field', async () => { + await securitySolutionApi + .createRule({ body: getCustomQueryRuleParams({ rule_id: 'custom-rule', version: 23 }) }) + .expect(200); + + const rule = getCustomQueryRuleParams({ rule_id: 'custom-rule', version: undefined }); + const { body } = await importRules([rule], true); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: 'custom-rule' }); + expect(importedRule).toMatchObject({ + rule_id: 'custom-rule', + version: 1, + rule_source: { type: 'internal' }, + immutable: false, + }); + }); + + it('imports a prebuilt rule with a non-existing rule_id', async () => { + const rule = createRuleAssetSavedObject({ rule_id: 'wacky-rule-id', version: 1234 })[ + 'security-rule' + ]; + const { body } = await importRules([rule]); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: 'wacky-rule-id' }); + expect(importedRule).toMatchObject({ + rule_id: 'wacky-rule-id', + version: 1234, + rule_source: { type: 'internal' }, + immutable: false, + }); + }); + it('rejects a versionless prebuilt rule', async () => { - const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: undefined }); + const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: undefined }); // Uses the `getCustomQueryRuleParams` util intead of the `createRuleAssetSavedObject` util because we are forcing an invalid rule body according to the Zod schema const { body } = await importRules([rule]); expect(body.errors).toHaveLength(1); @@ -112,6 +243,19 @@ export default ({ getService }: FtrProviderContext): void => { }); }); + it('rejects a prebuilt rule without a rule_id', async () => { + const rule = getCustomQueryRuleParams({ rule_id: undefined, version: 1 }); + const { body } = await importRules([rule]); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { + message: `rule_id: Required`, + status_code: 400, + }, + }); + }); + it('respects the version of a prebuilt rule', async () => { const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[1], version: 9999 }); const { body } = await importRules([rule]); @@ -135,9 +279,15 @@ export default ({ getService }: FtrProviderContext): void => { it('imports a combination of prebuilt and custom rules', async () => { const rules = [ getCustomQueryRuleParams({ rule_id: 'custom-rule', version: 23 }), - getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: 1234 }), getCustomQueryRuleParams({ rule_id: 'custom-rule-2', version: undefined }), - prebuiltRules[3], + // Unmodified prebuilt rule with matching rule_id and version + createRuleAssetSavedObject({ rule_id: 'rule-2', version: 2 })['security-rule'], + // Customized prebuilt rule with a matching rule_id and version + createRuleAssetSavedObject({ + rule_id: 'rule-1', + version: 2, + name: 'Customized prebuilt rule', + })['security-rule'], ]; const { body } = await importRules(rules); @@ -159,12 +309,6 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'internal' }, immutable: false, }), - expect.objectContaining({ - rule_id: prebuiltRuleIds[0], - version: 1234, - rule_source: { type: 'external', is_customized: true }, - immutable: true, - }), expect.objectContaining({ rule_id: 'custom-rule-2', version: 1, @@ -172,14 +316,37 @@ export default ({ getService }: FtrProviderContext): void => { immutable: false, }), expect.objectContaining({ - rule_id: prebuiltRules[3].rule_id, - version: prebuiltRules[3].version, + rule_id: 'rule-1', + version: 2, + rule_source: { type: 'external', is_customized: true }, + immutable: true, + }), + expect.objectContaining({ + rule_id: 'rule-2', + version: 2, rule_source: { type: 'external', is_customized: false }, immutable: true, }), ]) ); }); + + // TODO: Fix the test setup https://github.com/elastic/kibana/pull/206893#discussion_r1966170712 + it.skip('imports prebuilt rules when the rules package is not installed', async () => { + await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService }); // First we delete the rule package + + const { body } = await importRules([prebuiltRules[0]]); // Then we import a rule which should cause the rule package to be redownloaded + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const status = await getPrebuiltRulesAndTimelinesStatus(es, supertest); + expect(status.rules_installed).toEqual(1); // The rule package is now redownloaded and recognizes the rule_id as an installed rule + }); }); }); }; diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts b/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts index 4be7fb43cd11e..27a6dff7968e3 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts @@ -201,6 +201,7 @@ export const filterByElasticRules = () => { export const filterByCustomRules = () => { cy.get(CUSTOM_RULES_BTN).click(); + waitForRulesTableToBeRefreshed(); }; export const filterByEnabledRules = () => {