From dfb109843de36efce80e6b684f9a77fe96938be3 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 28 Jun 2022 10:29:55 +0100 Subject: [PATCH] Support ironbank docker build (#14298) This commit adds a rake task `rake artifact:dockerfile_ironbank` to generate ironbank docker build context for automatic release. The output can be found in build/logstash-ironbank-$VERSION-docker-build-context.tar.gz Co-authored-by: Rob Bavey --- .gitignore | 1 + docker/Makefile | 44 ++- docker/ironbank/LICENSE | 280 ++++++++++++++++++ docker/ironbank/README.md | 34 +++ docker/ironbank/go/src/env2yaml/go.mod | 5 + docker/ironbank/go/src/env2yaml/go.sum | 3 + .../go/src/env2yaml/vendor/modules.txt | 2 + docker/templates/Dockerfile.j2 | 94 +++++- docker/templates/hardening_manifest.yaml | 72 +++++ rakelib/artifacts.rake | 13 +- 10 files changed, 531 insertions(+), 17 deletions(-) create mode 100644 docker/ironbank/LICENSE create mode 100644 docker/ironbank/README.md create mode 100644 docker/ironbank/go/src/env2yaml/go.mod create mode 100644 docker/ironbank/go/src/env2yaml/go.sum create mode 100644 docker/ironbank/go/src/env2yaml/vendor/modules.txt create mode 100644 docker/templates/hardening_manifest.yaml diff --git a/.gitignore b/.gitignore index 4c0c15eb7c9..1b0d8f00054 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ out local test/setup/elasticsearch/elasticsearch-* vendor +!docker/ironbank/go/src/env2yaml/vendor .sass-cache /data .buildpath diff --git a/docker/Makefile b/docker/Makefile index 4b62c87cc6f..01028bb0ebd 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -84,7 +84,32 @@ docker_paths: mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml mkdir -p $(ARTIFACTS_DIR)/docker/pipeline -public-dockerfiles: public-dockerfiles_oss public_dockerfiles_full public_dockerfiles_ubi8 +COPY_IRONBANK_FILES = $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md + +$(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml: data/logstash/config/pipelines.yml +$(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml: data/logstash/config/logstash-full.yml +$(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties: data/logstash/config/log4j2.properties +$(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf: data/logstash/pipeline/default.conf +$(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint +$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go +$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod: ironbank/go/src/env2yaml/go.mod +$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum: ironbank/go/src/env2yaml/go.sum +$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt: ironbank/go/src/env2yaml/vendor/modules.txt +$(ARTIFACTS_DIR)/ironbank/LICENSE: ironbank/LICENSE +$(ARTIFACTS_DIR)/ironbank/README.md: ironbank/README.md + +$(ARTIFACTS_DIR)/ironbank/%: + cp -f $< $@ + +ironbank_docker_paths: + mkdir -p $(ARTIFACTS_DIR)/ironbank/ + mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts + mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/bin + mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/config + mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor + mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline + +public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_ubi8 public-dockerfiles_ironbank public-dockerfiles_full: venv templates/Dockerfile.j2 docker_paths $(COPY_FILES) jinja2 \ @@ -128,6 +153,23 @@ public-dockerfiles_ubi8: venv templates/Dockerfile.j2 docker_paths $(COPY_FILES) cp $(ARTIFACTS_DIR)/Dockerfile-ubi8 Dockerfile && \ tar -zcf ../logstash-ubi8-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline +public-dockerfiles_ironbank: templates/hardening_manifest.yaml templates/Dockerfile.j2 ironbank_docker_paths $(COPY_IRONBANK_FILES) + jinja2 \ + -D elastic_version='$(ELASTIC_VERSION)' \ + templates/hardening_manifest.yaml > $(ARTIFACTS_DIR)/ironbank/hardening_manifest.yaml && \ + jinja2 \ + -D created_date='$(BUILD_DATE)' \ + -D elastic_version='$(ELASTIC_VERSION)' \ + -D arch='${ARCHITECTURE}' \ + -D version_tag='$(VERSION_TAG)' \ + -D image_flavor='ironbank' \ + -D local_artifacts='false' \ + -D release='$(RELEASE)' \ + templates/Dockerfile.j2 > $(ARTIFACTS_DIR)/Dockerfile-ironbank && \ + cd $(ARTIFACTS_DIR)/ironbank && \ + cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \ + tar -zcf ../logstash-ironbank-$(VERSION_TAG)-docker-build-context.tar.gz scripts Dockerfile hardening_manifest.yaml LICENSE README.md + # Push the image to the dedicated push endpoint at "push.docker.elastic.co" push: $(foreach FLAVOR, $(IMAGE_FLAVORS), \ diff --git a/docker/ironbank/LICENSE b/docker/ironbank/LICENSE new file mode 100644 index 00000000000..632c3abe22e --- /dev/null +++ b/docker/ironbank/LICENSE @@ -0,0 +1,280 @@ +ELASTIC LICENSE AGREEMENT + +PLEASE READ CAREFULLY THIS ELASTIC LICENSE AGREEMENT (THIS "AGREEMENT"), WHICH +CONSTITUTES A LEGALLY BINDING AGREEMENT AND GOVERNS ALL OF YOUR USE OF ALL OF +THE ELASTIC SOFTWARE WITH WHICH THIS AGREEMENT IS INCLUDED ("ELASTIC SOFTWARE") +THAT IS PROVIDED IN OBJECT CODE FORMAT, AND, IN ACCORDANCE WITH SECTION 2 BELOW, +CERTAIN OF THE ELASTIC SOFTWARE THAT IS PROVIDED IN SOURCE CODE FORMAT. BY +INSTALLING OR USING ANY OF THE ELASTIC SOFTWARE GOVERNED BY THIS AGREEMENT, YOU +ARE ASSENTING TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE +WITH SUCH TERMS AND CONDITIONS, YOU MAY NOT INSTALL OR USE THE ELASTIC SOFTWARE +GOVERNED BY THIS AGREEMENT. IF YOU ARE INSTALLING OR USING THE SOFTWARE ON +BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL +AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF +SUCH ENTITY. + +Posted Date: April 20, 2018 + +This Agreement is entered into by and between Elasticsearch BV ("Elastic") and +You, or the legal entity on behalf of whom You are acting (as applicable, +"You"). + +1. OBJECT CODE END USER LICENSES, RESTRICTIONS AND THIRD PARTY OPEN SOURCE +SOFTWARE + + 1.1 Object Code End User License. Subject to the terms and conditions of + Section 1.2 of this Agreement, Elastic hereby grants to You, AT NO CHARGE and + for so long as you are not in breach of any provision of this Agreement, a + License to the Basic Features and Functions of the Elastic Software. + + 1.2 Reservation of Rights; Restrictions. As between Elastic and You, Elastic + and its licensors own all right, title and interest in and to the Elastic + Software, and except as expressly set forth in Sections 1.1, and 2.1 of this + Agreement, no other license to the Elastic Software is granted to You under + this Agreement, by implication, estoppel or otherwise. You agree not to: (i) + reverse engineer or decompile, decrypt, disassemble or otherwise reduce any + Elastic Software provided to You in Object Code, or any portion thereof, to + Source Code, except and only to the extent any such restriction is prohibited + by applicable law, (ii) except as expressly permitted in this Agreement, + prepare derivative works from, modify, copy or use the Elastic Software Object + Code or the Commercial Software Source Code in any manner; (iii) except as + expressly permitted in Section 1.1 above, transfer, sell, rent, lease, + distribute, sublicense, loan or otherwise transfer, Elastic Software Object + Code, in whole or in part, to any third party; (iv) use Elastic Software + Object Code for providing time-sharing services, any software-as-a-service, + service bureau services or as part of an application services provider or + other service offering (collectively, "SaaS Offering") where obtaining access + to the Elastic Software or the features and functions of the Elastic Software + is a primary reason or substantial motivation for users of the SaaS Offering + to access and/or use the SaaS Offering ("Prohibited SaaS Offering"); (v) + circumvent the limitations on use of Elastic Software provided to You in + Object Code format that are imposed or preserved by any License Key, or (vi) + alter or remove any Marks and Notices in the Elastic Software. If You have any + question as to whether a specific SaaS Offering constitutes a Prohibited SaaS + Offering, or are interested in obtaining Elastic's permission to engage in + commercial or non-commercial distribution of the Elastic Software, please + contact elastic_license@elastic.co. + + 1.3 Third Party Open Source Software. The Commercial Software may contain or + be provided with third party open source libraries, components, utilities and + other open source software (collectively, "Open Source Software"), which Open + Source Software may have applicable license terms as identified on a website + designated by Elastic. Notwithstanding anything to the contrary herein, use of + the Open Source Software shall be subject to the license terms and conditions + applicable to such Open Source Software, to the extent required by the + applicable licensor (which terms shall not restrict the license rights granted + to You hereunder, but may contain additional rights). To the extent any + condition of this Agreement conflicts with any license to the Open Source + Software, the Open Source Software license will govern with respect to such + Open Source Software only. Elastic may also separately provide you with + certain open source software that is licensed by Elastic. Your use of such + Elastic open source software will not be governed by this Agreement, but by + the applicable open source license terms. + +2. COMMERCIAL SOFTWARE SOURCE CODE + + 2.1 Limited License. Subject to the terms and conditions of Section 2.2 of + this Agreement, Elastic hereby grants to You, AT NO CHARGE and for so long as + you are not in breach of any provision of this Agreement, a limited, + non-exclusive, non-transferable, fully paid up royalty free right and license + to the Commercial Software in Source Code format, without the right to grant + or authorize sublicenses, to prepare Derivative Works of the Commercial + Software, provided You (i) do not hack the licensing mechanism, or otherwise + circumvent the intended limitations on the use of Elastic Software to enable + features other than Basic Features and Functions or those features You are + entitled to as part of a Subscription, and (ii) use the resulting object code + only for reasonable testing purposes. + + 2.2 Restrictions. Nothing in Section 2.1 grants You the right to (i) use the + Commercial Software Source Code other than in accordance with Section 2.1 + above, (ii) use a Derivative Work of the Commercial Software outside of a + Non-production Environment, in any production capacity, on a temporary or + permanent basis, or (iii) transfer, sell, rent, lease, distribute, sublicense, + loan or otherwise make available the Commercial Software Source Code, in whole + or in part, to any third party. Notwithstanding the foregoing, You may + maintain a copy of the repository in which the Source Code of the Commercial + Software resides and that copy may be publicly accessible, provided that you + include this Agreement with Your copy of the repository. + +3. TERMINATION + + 3.1 Termination. This Agreement will automatically terminate, whether or not + You receive notice of such Termination from Elastic, if You breach any of its + provisions. + + 3.2 Post Termination. Upon any termination of this Agreement, for any reason, + You shall promptly cease the use of the Elastic Software in Object Code format + and cease use of the Commercial Software in Source Code format. For the + avoidance of doubt, termination of this Agreement will not affect Your right + to use Elastic Software, in either Object Code or Source Code formats, made + available under the Apache License Version 2.0. + + 3.3 Survival. Sections 1.2, 2.2. 3.3, 4 and 5 shall survive any termination or + expiration of this Agreement. + +4. DISCLAIMER OF WARRANTIES AND LIMITATION OF LIABILITY + + 4.1 Disclaimer of Warranties. TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE + LAW, THE ELASTIC SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, + AND ELASTIC AND ITS LICENSORS MAKE NO WARRANTIES WHETHER EXPRESSED, IMPLIED OR + STATUTORY REGARDING OR RELATING TO THE ELASTIC SOFTWARE. TO THE MAXIMUM EXTENT + PERMITTED UNDER APPLICABLE LAW, ELASTIC AND ITS LICENSORS SPECIFICALLY + DISCLAIM ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NON-INFRINGEMENT WITH RESPECT TO THE ELASTIC SOFTWARE, AND WITH + RESPECT TO THE USE OF THE FOREGOING. FURTHER, ELASTIC DOES NOT WARRANT RESULTS + OF USE OR THAT THE ELASTIC SOFTWARE WILL BE ERROR FREE OR THAT THE USE OF THE + ELASTIC SOFTWARE WILL BE UNINTERRUPTED. + + 4.2 Limitation of Liability. IN NO EVENT SHALL ELASTIC OR ITS LICENSORS BE + LIABLE TO YOU OR ANY THIRD PARTY FOR ANY DIRECT OR INDIRECT DAMAGES, + INCLUDING, WITHOUT LIMITATION, FOR ANY LOSS OF PROFITS, LOSS OF USE, BUSINESS + INTERRUPTION, LOSS OF DATA, COST OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY + SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, IN CONNECTION WITH + OR ARISING OUT OF THE USE OR INABILITY TO USE THE ELASTIC SOFTWARE, OR THE + PERFORMANCE OF OR FAILURE TO PERFORM THIS AGREEMENT, WHETHER ALLEGED AS A + BREACH OF CONTRACT OR TORTIOUS CONDUCT, INCLUDING NEGLIGENCE, EVEN IF ELASTIC + HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +5. MISCELLANEOUS + + This Agreement completely and exclusively states the entire agreement of the + parties regarding the subject matter herein, and it supersedes, and its terms + govern, all prior proposals, agreements, or other communications between the + parties, oral or written, regarding such subject matter. This Agreement may be + modified by Elastic from time to time, and any such modifications will be + effective upon the "Posted Date" set forth at the top of the modified + Agreement. If any provision hereof is held unenforceable, this Agreement will + continue without said provision and be interpreted to reflect the original + intent of the parties. This Agreement and any non-contractual obligation + arising out of or in connection with it, is governed exclusively by Dutch law. + This Agreement shall not be governed by the 1980 UN Convention on Contracts + for the International Sale of Goods. All disputes arising out of or in + connection with this Agreement, including its existence and validity, shall be + resolved by the courts with jurisdiction in Amsterdam, The Netherlands, except + where mandatory law provides for the courts at another location in The + Netherlands to have jurisdiction. The parties hereby irrevocably waive any and + all claims and defenses either might otherwise have in any such action or + proceeding in any of such courts based upon any alleged lack of personal + jurisdiction, improper venue, forum non conveniens or any similar claim or + defense. A breach or threatened breach, by You of Section 2 may cause + irreparable harm for which damages at law may not provide adequate relief, and + therefore Elastic shall be entitled to seek injunctive relief without being + required to post a bond. You may not assign this Agreement (including by + operation of law in connection with a merger or acquisition), in whole or in + part to any third party without the prior written consent of Elastic, which + may be withheld or granted by Elastic in its sole and absolute discretion. + Any assignment in violation of the preceding sentence is void. Notices to + Elastic may also be sent to legal@elastic.co. + +6. DEFINITIONS + + The following terms have the meanings ascribed: + + 6.1 "Affiliate" means, with respect to a party, any entity that controls, is + controlled by, or which is under common control with, such party, where + "control" means ownership of at least fifty percent (50%) of the outstanding + voting shares of the entity, or the contractual right to establish policy for, + and manage the operations of, the entity. + + 6.2 "Basic Features and Functions" means those features and functions of the + Elastic Software that are eligible for use under a Basic license, as set forth + at https://www.elastic.co/subscriptions, as may be modified by Elastic from + time to time. + + 6.3 "Commercial Software" means the Elastic Software Source Code in any file + containing a header stating the contents are subject to the Elastic License or + which is contained in the repository folder labeled "x-pack", unless a LICENSE + file present in the directory subtree declares a different license. + + 6.4 "Derivative Work of the Commercial Software" means, for purposes of this + Agreement, any modification(s) or enhancement(s) to the Commercial Software, + which represent, as a whole, an original work of authorship. + + 6.5 "License" means a limited, non-exclusive, non-transferable, fully paid up, + royalty free, right and license, without the right to grant or authorize + sublicenses, solely for Your internal business operations to (i) install and + use the applicable Features and Functions of the Elastic Software in Object + Code, and (ii) permit Contractors and Your Affiliates to use the Elastic + software as set forth in (i) above, provided that such use by Contractors must + be solely for Your benefit and/or the benefit of Your Affiliates, and You + shall be responsible for all acts and omissions of such Contractors and + Affiliates in connection with their use of the Elastic software that are + contrary to the terms and conditions of this Agreement. + + 6.6 "License Key" means a sequence of bytes, including but not limited to a + JSON blob, that is used to enable certain features and functions of the + Elastic Software. + + 6.7 "Marks and Notices" means all Elastic trademarks, trade names, logos and + notices present on the Documentation as originally provided by Elastic. + + 6.8 "Non-production Environment" means an environment for development, testing + or quality assurance, where software is not used for production purposes. + + 6.9 "Object Code" means any form resulting from mechanical transformation or + translation of Source Code form, including but not limited to compiled object + code, generated documentation, and conversions to other media types. + + 6.10 "Source Code" means the preferred form of computer software for making + modifications, including but not limited to software source code, + documentation source, and configuration files. + + 6.11 "Subscription" means the right to receive Support Services and a License + to the Commercial Software. + + +GOVERNMENT END USER ADDENDUM TO THE ELASTIC LICENSE AGREEMENT + + This ADDENDUM TO THE ELASTIC LICENSE AGREEMENT (this "Addendum") applies +only to U.S. Federal Government, State Government, and Local Government +entities ("Government End Users") of the Elastic Software. This Addendum is +subject to, and hereby incorporated into, the Elastic License Agreement, +which is being entered into as of even date herewith, by Elastic and You (the +"Agreement"). This Addendum sets forth additional terms and conditions +related to Your use of the Elastic Software. Capitalized terms not defined in +this Addendum have the meaning set forth in the Agreement. + + 1. LIMITED LICENSE TO DISTRIBUTE (DSOP ONLY). Subject to the terms and +conditions of the Agreement (including this Addendum), Elastic grants the +Department of Defense Enterprise DevSecOps Initiative (DSOP) a royalty-free, +non-exclusive, non-transferable, limited license to reproduce and distribute +the Elastic Software solely through a software distribution repository +controlled and managed by DSOP, provided that DSOP: (i) distributes the +Elastic Software complete and unmodified, inclusive of the Agreement +(including this Addendum) and (ii) does not remove or alter any proprietary +legends or notices contained in the Elastic Software. + + 2. CHOICE OF LAW. The choice of law and venue provisions set forth shall +prevail over those set forth in Section 5 of the Agreement. + + "For U.S. Federal Government Entity End Users. This Agreement and any + non-contractual obligation arising out of or in connection with it, is + governed exclusively by U.S. Federal law. To the extent permitted by + federal law, the laws of the State of Delaware (excluding Delaware choice + of law rules) will apply in the absence of applicable federal law. + + For State and Local Government Entity End Users. This Agreement and any + non-contractual obligation arising out of or in connection with it, is + governed exclusively by the laws of the state in which you are located + without reference to conflict of laws. Furthermore, the Parties agree that + the Uniform Computer Information Transactions Act or any version thereof, + adopted by any state in any form ('UCITA'), shall not apply to this + Agreement and, to the extent that UCITA is applicable, the Parties agree to + opt out of the applicability of UCITA pursuant to the opt-out provision(s) + contained therein." + + 3. ELASTIC LICENSE MODIFICATION. Section 5 of the Agreement is hereby +amended to replace + + "This Agreement may be modified by Elastic from time to time, and any + such modifications will be effective upon the "Posted Date" set forth at + the top of the modified Agreement." + + with: + + "This Agreement may be modified by Elastic from time to time; provided, + however, that any such modifications shall apply only to Elastic Software + that is installed after the "Posted Date" set forth at the top of the + modified Agreement." + +V100820.0 diff --git a/docker/ironbank/README.md b/docker/ironbank/README.md new file mode 100644 index 00000000000..9fade56b5b6 --- /dev/null +++ b/docker/ironbank/README.md @@ -0,0 +1,34 @@ +# Logstash +Logstash is part of the [Elastic Stack](https://www.elastic.co/products) along with Elasticsearch, Kibana, and Beats. Logstash is a server-side data processing pipeline that ingests data from a multitude of sources simultaneously, transforms it, and then sends it to your favorite "stash." (Ours is Elasticsearch, naturally.). Logstash has over 200 plugins, and you can write your own very easily as well. + +For more info, see + +### Installation instructions + +Please follow the documentation on [how to install Logstash with Docker](https://www.elastic.co/guide/en/logstash/current/docker.html). + +## Documentation and Getting Started + +You can find the documentation and getting started guides for Logstash +on the [elastic.co site](https://www.elastic.co/guide/en/logstash/current/getting-started-with-logstash.html). + +### Where to file issues and PRs + +- [Issues](https://github.com/elastic/logstash/issues) +- [PRs](https://github.com/elastic/logstash/pulls) + +**Please open new issues and pull requests for plugins under its own repository** + +For example, if you have to report an issue/enhancement for the Elasticsearch output, please do so [here](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues). + +## Need Help? + +- [Logstash Forum](https://discuss.elastic.co/c/logstash) +- [Logstash Documentation](https://www.elastic.co/guide/en/logstash/current/index.html) +- [Elastic Support](https://www.elastic.co/subscriptions) + +## Project Principles + +* Community: If a newbie has a bad time, it's a bug. +* Software: Make it work, then make it right, then make it fast. +* Technology: If it doesn't do a thing today, we can make it do it tomorrow. diff --git a/docker/ironbank/go/src/env2yaml/go.mod b/docker/ironbank/go/src/env2yaml/go.mod new file mode 100644 index 00000000000..a21d1f1af29 --- /dev/null +++ b/docker/ironbank/go/src/env2yaml/go.mod @@ -0,0 +1,5 @@ +module env2yaml + +go 1.13 + +require gopkg.in/yaml.v2 v2.3.0 diff --git a/docker/ironbank/go/src/env2yaml/go.sum b/docker/ironbank/go/src/env2yaml/go.sum new file mode 100644 index 00000000000..8fabe8daafe --- /dev/null +++ b/docker/ironbank/go/src/env2yaml/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/docker/ironbank/go/src/env2yaml/vendor/modules.txt b/docker/ironbank/go/src/env2yaml/vendor/modules.txt new file mode 100644 index 00000000000..bdd2db2d793 --- /dev/null +++ b/docker/ironbank/go/src/env2yaml/vendor/modules.txt @@ -0,0 +1,2 @@ +# gopkg.in/yaml.v2 v2.3.0 +gopkg.in/yaml.v2 diff --git a/docker/templates/Dockerfile.j2 b/docker/templates/Dockerfile.j2 index 1b38cc958ea..76510dbc703 100644 --- a/docker/templates/Dockerfile.j2 +++ b/docker/templates/Dockerfile.j2 @@ -18,35 +18,83 @@ {% set package_manager = 'microdnf' -%} # Minimal distributions do not ship with en language packs. {% set locale = 'C.UTF-8' -%} +{% elif image_flavor == 'ironbank' -%} + {% set package_manager = 'yum' -%} {% else -%} {% set base_image = 'ubuntu:20.04' -%} {% set package_manager = 'apt-get' -%} {% set locale = 'en_US.UTF-8' -%} {% endif -%} + +{% if image_flavor == 'ironbank' -%} +ARG BASE_REGISTRY=registry1.dsop.io +ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8 +ARG BASE_TAG=8.6 +ARG LOGSTASH_VERSION={{ elastic_version }} +ARG GOLANG_VERSION=1.17.8 + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS env2yaml + +ARG GOLANG_VERSION + +# install golang +RUN yum update -y && yum install -y git +COPY go${GOLANG_VERSION}.linux-amd64.tar.gz /opt/go.tar.gz +RUN tar -C /usr/local -xzf /opt/go.tar.gz +ENV PATH=$PATH:/usr/local/go/bin + +# compile the env2yaml tool +COPY v2.3.0.tar.gz /opt/env2yaml.tar.gz +COPY scripts/go /usr/local/src/go +WORKDIR /usr/local/src/go/src/env2yaml +RUN mkdir -p vendor/gopkg.in +RUN tar -zxf /opt/env2yaml.tar.gz -C vendor/gopkg.in +RUN mv vendor/gopkg.in/yaml-2.3.0 vendor/gopkg.in/yaml.v2 +ENV GOPATH=/usr/local/src/go +RUN go build -mod vendor + +# stage 1: unpack logstash +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS builder + +ARG LOGSTASH_VERSION + +WORKDIR /usr/share/ +COPY logstash-${LOGSTASH_VERSION}-linux-x86_64.tar.gz /opt/logstash.tar.gz + +RUN tar zxf /opt/logstash.tar.gz && \ + mv /usr/share/logstash-${LOGSTASH_VERSION} /usr/share/logstash + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +{% else -%} FROM {{ base_image }} +{% endif -%} + RUN for iter in {1..10}; do \ -{% if image_flavor != 'ubi8' -%} +{% if image_flavor == 'full' or image_flavor == 'oss' -%} export DEBIAN_FRONTEND=noninteractive && \ {% endif -%} {{ package_manager }} update -y && \ +{% if image_flavor != 'ironbank' -%} {{ package_manager }} upgrade -y && \ +{% endif -%} {{ package_manager }} install -y procps findutils tar gzip curl && \ -{% if image_flavor == 'ubi8' -%} +{% if image_flavor == 'ubi8' or image_flavor == 'ironbank' -%} {{ package_manager }} install -y which shadow-utils && \ {% else -%} {{ package_manager }} install -y locales && \ {% endif -%} {{ package_manager }} clean all && \ -{% if image_flavor != 'ubi8' -%} +{% if image_flavor == 'full' or image_flavor == 'oss' -%} locale-gen 'en_US.UTF-8' && \ {{ package_manager }} clean metadata && \ {% endif -%} exit_code=0 && break || exit_code=$? && \ echo "packaging error: retry $iter in 10s" && \ {{ package_manager }} clean all && \ -{% if image_flavor != 'ubi8' -%} +{% if image_flavor == 'full' or image_flavor == 'oss' -%} {{ package_manager }} clean metadata && \ {% endif -%} sleep 10; done; \ @@ -55,13 +103,21 @@ RUN for iter in {1..10}; do \ # Provide a non-root user to run the process. RUN groupadd --gid 1000 logstash && \ adduser --uid 1000 --gid 1000 \ - --home /usr/share/logstash --no-create-home \ + {% if image_flavor != 'ironbank' %} --home {% else %} --home-dir {% endif -%} /usr/share/logstash --no-create-home \ logstash +{% if image_flavor == 'ironbank' -%} +WORKDIR /usr/share/logstash +COPY --from=env2yaml /usr/local/src/go/src/env2yaml/env2yaml /usr/local/bin/env2yaml +COPY --from=builder --chown=1000:0 /usr/share/logstash /usr/share/logstash +{% endif -%} + # Add Logstash itself. -RUN curl -Lo - {{ url_root }}/{{ tarball }} | \ +RUN \ +{% if image_flavor != 'ironbank' -%} curl -Lo - {{ url_root }}/{{ tarball }} | \ tar zxf - -C /usr/share && \ mv /usr/share/logstash-{{ elastic_version }} /usr/share/logstash && \ +{% endif -%} chown --recursive logstash:logstash /usr/share/logstash/ && \ chown -R logstash:root /usr/share/logstash && \ chmod -R g=u /usr/share/logstash && \ @@ -71,14 +127,15 @@ RUN curl -Lo - {{ url_root }}/{{ tarball }} | \ find /usr/share/logstash -type d -exec chmod g+s {} \; && \ ln -s /usr/share/logstash /opt/logstash - +{% if image_flavor != 'ironbank' -%} WORKDIR /usr/share/logstash - +{% endif -%} ENV ELASTIC_CONTAINER true ENV PATH=/usr/share/logstash/bin:$PATH # Provide a minimal configuration, so that simple invocations will provide # a good experience. +{% if image_flavor != 'ironbank' -%} ADD config/pipelines.yml config/pipelines.yml {% if image_flavor == 'oss' -%} ADD config/logstash-oss.yml config/logstash.yml @@ -88,21 +145,28 @@ ADD config/logstash-full.yml config/logstash.yml ADD config/log4j2.properties config/ ADD pipeline/default.conf pipeline/logstash.conf RUN chown --recursive logstash:root config/ pipeline/ - # Ensure Logstash gets the correct locale by default. ENV LANG={{ locale }} LC_ALL={{ locale }} - +ADD env2yaml/env2yaml /usr/local/bin/ # Place the startup wrapper script. ADD bin/docker-entrypoint /usr/local/bin/ +{% else -%} +COPY scripts/config/pipelines.yml config/pipelines.yml +COPY scripts/config/logstash.yml config/logstash.yml +COPY scripts/config/log4j2.properties config/ +COPY scripts/pipeline/default.conf pipeline/logstash.conf +RUN chown --recursive logstash:root config/ pipeline/ +# Place the startup wrapper script. +COPY scripts/bin/docker-entrypoint /usr/local/bin/ +{% endif -%} + RUN chmod 0755 /usr/local/bin/docker-entrypoint USER 1000 -ADD env2yaml/env2yaml /usr/local/bin/ - EXPOSE 9600 5044 - +{% if image_flavor != 'ironbank' -%} LABEL org.label-schema.schema-version="1.0" \ org.label-schema.vendor="Elastic" \ org.opencontainers.image.vendor="Elastic" \ @@ -125,6 +189,10 @@ LABEL org.label-schema.schema-version="1.0" \ vendor="Elastic" \ {% endif -%} org.opencontainers.image.created={{ created_date }} +{% endif -%} +{% if image_flavor == 'ironbank' -%} +HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9600 || exit 1 +{% endif -%} ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] diff --git a/docker/templates/hardening_manifest.yaml b/docker/templates/hardening_manifest.yaml new file mode 100644 index 00000000000..3876ac33964 --- /dev/null +++ b/docker/templates/hardening_manifest.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: v1 + +# The repository name in registry1, excluding /ironbank/ +name: "elastic/logstash/logstash" + +# List of tags to push for the repository in registry1 +# The most specific version should be the first tag and will be shown +# on ironbank.dsop.io +tags: +- "{{ elastic_version }}" +- "latest" + +# Build args passed to Dockerfile ARGs +args: + BASE_IMAGE: "redhat/ubi/ubi8" + BASE_TAG: "8.6" + LOGSTASH_VERSION: "{{ elastic_version }}" + GOLANG_VERSION: "1.17.8" + +# Docker image labels +labels: + org.opencontainers.image.title: "logstash" + ## Human-readable description of the software packaged in the image + org.opencontainers.image.description: "Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" + ## License(s) under which contained software is distributed + org.opencontainers.image.licenses: "Elastic License" + ## URL to find more information on the image + org.opencontainers.image.url: "https://www.elastic.co/products/logstash" + ## Name of the distributing entity, organization or individual + org.opencontainers.image.vendor: "Elastic" + org.opencontainers.image.version: "{{ elastic_version }}" + ## Keywords to help with search (ex. "cicd,gitops,golang") + # mil.dso.ironbank.image.keywords: "FIXME" + ## This value can be "opensource" or "commercial" + # mil.dso.ironbank.image.type: "FIXME" + ## Product the image belongs to for grouping multiple images + mil.dso.ironbank.product.name: "Logstash" + +# List of resources to make available to the offline build context +resources: +- filename: logstash-{{ elastic_version }}-linux-x86_64.tar.gz + url: https://artifacts.elastic.co/downloads/logstash/logstash-{{ elastic_version }}-linux-x86_64.tar.gz + validation: + type: sha512 + value: +- filename: go1.17.8.linux-amd64.tar.gz + url: https://dl.google.com/go/go1.17.8.linux-amd64.tar.gz + validation: + type: sha256 + value: 980e65a863377e69fd9b67df9d8395fd8e93858e7a24c9f55803421e453f4f99 +- filename: v2.3.0.tar.gz + url: https://github.com/go-yaml/yaml/archive/v2.3.0.tar.gz + validation: + type: sha512 + value: ba934e9cb5ebd2346d3897308b71d13bc6471a8dbc0dc0d46a02644ee6b6553d20c20393471b81025b572a9b03e3326bde9c3e8be156474f1a1f91ff027b6a4f + +# List of project maintainers +maintainers: +- name: "Nassim Kammah" + username: "nkammah" + email: "nassim.kammah@elastic.co" +- name: "Joao Duarte" + username: "joaodiasduarte" + email: "joao@elastic.co" +- name: "Rob Bavey" + username: "robbavey" + email: "rob.bavey@elastic.co" +- name: "Kaise Cheng" + username: "kaisecheng" + email: "kaise.cheng@elastic.co" + diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index e8cfaa8a3f7..93023af6600 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -293,6 +293,7 @@ namespace "artifact" do build_dockerfile('oss') build_dockerfile('full') build_dockerfile('ubi8') + build_dockerfile('ironbank') end desc "Generate Dockerfile for oss images" @@ -303,16 +304,22 @@ namespace "artifact" do desc "Generate Dockerfile for full images" task "dockerfile_full" => ["prepare", "generate_build_metadata"] do - puts("[dockerfiles] Building default Dockerfiles") + puts("[dockerfiles] Building full Dockerfiles") build_dockerfile('full') end desc "Generate Dockerfile for full images" task "dockerfile_ubi8" => ["prepare", "generate_build_metadata"] do - puts("[dockerfiles] Building default Dockerfiles") + puts("[dockerfiles] Building ubi8 Dockerfiles") build_dockerfile('ubi8') end + desc "Generate build context for ironbank" + task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do + puts("[dockerfiles] Building ironbank Dockerfiles") + build_dockerfile('ironbank') + end + # Auxiliary tasks task "build" => [:generate_build_metadata] do Rake::Task["artifact:gems"].invoke unless SNAPSHOT_BUILD @@ -729,7 +736,7 @@ namespace "artifact" do } Dir.chdir("docker") do |dir| system(env, "make public-dockerfiles_#{flavor}") - puts "Dockerfiles created in #{::File.join(env['ARTIFACTS_DIR'], 'docker')}" + puts "Dockerfiles created in #{env['ARTIFACTS_DIR']}" end end end