From f526b47627d39686f3fc8afbf4500479e99301fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Mon, 3 Feb 2025 18:07:36 +0100 Subject: [PATCH] Remove dangling file for ASL integration (#665) --- .../amazon-security-lake/CONTRIBUTING.md | 6 + .../aws-lambda.dockerfile | 17 --- .../logstash/pipeline/indexer-to-file.conf | 34 ----- integrations/docker/amazon-security-lake.yml | 143 ------------------ 4 files changed, 6 insertions(+), 194 deletions(-) delete mode 100644 integrations/amazon-security-lake/aws-lambda.dockerfile delete mode 100644 integrations/amazon-security-lake/logstash/pipeline/indexer-to-file.conf delete mode 100644 integrations/docker/amazon-security-lake.yml diff --git a/integrations/amazon-security-lake/CONTRIBUTING.md b/integrations/amazon-security-lake/CONTRIBUTING.md index 1d8132d814c73..e819fac2f6c7e 100644 --- a/integrations/amazon-security-lake/CONTRIBUTING.md +++ b/integrations/amazon-security-lake/CONTRIBUTING.md @@ -16,6 +16,12 @@ This Docker Compose project will bring up these services: - our [events generator](../tools/events-generator/README.md) - an AWS Lambda Python container. +| Service | Address | Credentials | +| ------------- | ------------------------ | --------------- | +| Wazuh Indexer | https://localhost:9200 | admin:admin | +| Dashboards | https://localhost:5601 | admin:admin | +| S3 Ninja | http://localhost:9444/ui | | + On the one hand, the event generator will push events constantly to the indexer, to the `wazuh-alerts-4.x-sample` index by default (refer to the [events generator](../tools/events-generator/README.md) documentation for customization options). On the other hand, Logstash will query for new data and deliver it to output configured in the pipeline `indexer-to-s3`. This pipeline delivers the data to an S3 bucket, from which the data is processed using a Lambda function, to finally be sent to the Amazon Security Lake bucket in Parquet format. The pipeline starts automatically, but if you need to start it manually, attach a terminal to the Logstash container and start the integration using the command below: diff --git a/integrations/amazon-security-lake/aws-lambda.dockerfile b/integrations/amazon-security-lake/aws-lambda.dockerfile deleted file mode 100644 index 7039c2b935de8..0000000000000 --- a/integrations/amazon-security-lake/aws-lambda.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# docker build --platform linux/amd64 --no-cache -f aws-lambda.dockerfile -t docker-image:test . -# docker run --platform linux/amd64 -p 9000:8080 docker-image:test - -# FROM public.ecr.aws/lambda/python:3.9 -FROM amazon/aws-lambda-python:3.12 - -# Copy requirements.txt -COPY requirements.aws.txt ${LAMBDA_TASK_ROOT} - -# Install the specified packages -RUN pip install -r requirements.aws.txt - -# Copy function code -COPY src ${LAMBDA_TASK_ROOT} - -# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) -CMD [ "lambda_function.lambda_handler" ] \ No newline at end of file diff --git a/integrations/amazon-security-lake/logstash/pipeline/indexer-to-file.conf b/integrations/amazon-security-lake/logstash/pipeline/indexer-to-file.conf deleted file mode 100644 index 1bee9afc62450..0000000000000 --- a/integrations/amazon-security-lake/logstash/pipeline/indexer-to-file.conf +++ /dev/null @@ -1,34 +0,0 @@ -input { - opensearch { - hosts => ["wazuh.indexer:9200"] - user => "${INDEXER_USERNAME}" - password => "${INDEXER_PASSWORD}" - ssl => true - ca_file => "/usr/share/logstash/root-ca.pem" - index => "wazuh-alerts-4.x-*" - query => '{ - "query": { - "range": { - "@timestamp": { - "gt": "now-1m" - } - } - } - }' - schedule => "* * * * *" - } -} - - -output { - stdout { - id => "output.stdout" - codec => json_lines - } - file { - id => "output.file" - path => "/var/log/logstash/indexer-to-file-%{+YYYY-MM-dd-HH}.log" - file_mode => 0644 - codec => json_lines - } -} diff --git a/integrations/docker/amazon-security-lake.yml b/integrations/docker/amazon-security-lake.yml deleted file mode 100644 index c02b51dab8c58..0000000000000 --- a/integrations/docker/amazon-security-lake.yml +++ /dev/null @@ -1,143 +0,0 @@ -version: "3.8" -name: "amazon-security-lake" - -services: - events-generator: - image: wazuh/indexer-events-generator - build: - context: ../tools/events-generator - container_name: events-generator - depends_on: - wazuh.indexer: - condition: service_healthy - command: bash -c "python run.py -a wazuh.indexer" - - wazuh.indexer: - image: opensearchproject/opensearch:2.12.0 - container_name: wazuh.indexer - depends_on: - wazuh-certs-generator: - condition: service_completed_successfully - hostname: wazuh.indexer - ports: - - 9200:9200 - environment: - # - cluster.name=opensearch-cluster - - node.name=wazuh.indexer - - discovery.type=single-node - # - cluster.initial_cluster_manager_nodes=opensearch-node - - bootstrap.memory_lock=true - - "DISABLE_INSTALL_DEMO_CONFIG=true" - - plugins.security.ssl.http.enabled=true - - plugins.security.allow_default_init_securityindex=true - - plugins.security.ssl.http.pemcert_filepath=/usr/share/opensearch/config/wazuh.indexer.pem - - plugins.security.ssl.transport.pemcert_filepath=/usr/share/opensearch/config/wazuh.indexer.pem - - plugins.security.ssl.http.pemkey_filepath=/usr/share/opensearch/config/wazuh.indexer-key.pem - - plugins.security.ssl.transport.pemkey_filepath=/usr/share/opensearch/config/wazuh.indexer-key.pem - - plugins.security.ssl.http.pemtrustedcas_filepath=/usr/share/opensearch/config/root-ca.pem - - plugins.security.ssl.transport.pemtrustedcas_filepath=/usr/share/opensearch/config/root-ca.pem - - plugins.security.authcz.admin_dn="CN=wazuh.indexer,OU=Wazuh,O=Wazuh,L=California, C=US" - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - healthcheck: - test: curl -sku admin:admin https://localhost:9200/_cat/health | grep -q docker-cluster - start_period: 10s - start_interval: 3s - volumes: - - data:/usr/share/opensearch/data - - ./certs/wazuh.indexer.pem:/usr/share/opensearch/config/wazuh.indexer.pem - - ./certs/wazuh.indexer-key.pem:/usr/share/opensearch/config/wazuh.indexer-key.pem - - ./certs/root-ca.pem:/usr/share/opensearch/config/root-ca.pem - - wazuh.dashboard: - image: opensearchproject/opensearch-dashboards:2.12.0 - container_name: wazuh.dashboard - depends_on: - - wazuh.indexer - hostname: wazuh.dashboard - ports: - - 5601:5601 # Map host port 5601 to container port 5601 - expose: - - "5601" # Expose port 5601 for web access to OpenSearch Dashboards - environment: - OPENSEARCH_HOSTS: '["https://wazuh.indexer:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query - - wazuh.integration.security.lake: - image: wazuh/indexer-security-lake-integration - build: - context: ../amazon-security-lake - container_name: wazuh.integration.security.lake - depends_on: - - wazuh.indexer - hostname: wazuh.integration.security.lake - environment: - LOG_LEVEL: trace - LOGSTASH_KEYSTORE_PASS: "SecretPassword" - MONITORING_ENABLED: false - AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" - AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - AWS_REGION: "us-east-1" - S3_BUCKET_RAW: "wazuh-aws-security-lake-raw" - AWS_ENDPOINT: "http://s3.ninja:9000" - ports: - - "5000:5000/tcp" - - "5000:5000/udp" - - "5044:5044" - - "9600:9600" - volumes: - - ../amazon-security-lake/logstash/pipeline:/usr/share/logstash/pipeline # TODO has 1000:1000. logstash's uid is 999 - - ./certs/root-ca.pem:/usr/share/logstash/root-ca.pem - - ../amazon-security-lake/src:/usr/share/logstash/amazon-security-lake # TODO use dedicated folder - # - ./credentials:/usr/share/logstash/.aws/credentials # TODO credentials are not commited (missing) - command: tail -f /var/log/logstash/logstash-plain.log - - s3.ninja: - image: scireum/s3-ninja:latest - container_name: s3.ninja - hostname: s3.ninja - ports: - - "9444:9000" - volumes: - - s3-data:/home/sirius/data - - aws.lambda: - image: wazuh/indexer-security-lake-integration:lambda - build: - context: ../amazon-security-lake - dockerfile: ../amazon-security-lake/aws-lambda.dockerfile - container_name: wazuh.integration.security.lake.aws.lambda - hostname: wazuh.integration.security.lake.aws.lambda - environment: - AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" - AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - AWS_REGION: "us-east-1" - AWS_BUCKET: "wazuh-aws-security-lake-parquet" - S3_BUCKET_OCSF: "wazuh-aws-security-lake-ocsf" - AWS_ENDPOINT: "http://s3.ninja:9000" - SOURCE_LOCATION: "wazuh" - ACCOUNT_ID: "111111111111" - IS_DEV: true - OCSF_CLASS: SECURITY_FINDING - volumes: - - ../amazon-security-lake/src:/var/task - ports: - - "9000:8080" - - wazuh-certs-generator: - image: wazuh/wazuh-certs-generator:0.0.1 - hostname: wazuh-certs-generator - container_name: wazuh-certs-generator - entrypoint: sh -c "/entrypoint.sh; chown -R 1000:999 /certificates; chmod 740 /certificates; chmod 440 /certificates/*" - volumes: - - ./certs/:/certificates/ - - ./config/certs.yml:/config/certs.yml - -volumes: - data: - s3-data: