Skip to content

Commit

Permalink
Support ironbank docker build (#14298)
Browse files Browse the repository at this point in the history
This commit adds a rake task `rake artifact:dockerfile_ironbank` to generate ironbank docker build context for automatic release.
The output can be found in build/logstash-ironbank-$VERSION-docker-build-context.tar.gz

Co-authored-by: Rob Bavey <rob.bavey@elastic.co>
  • Loading branch information
kaisecheng and robbavey authored Jun 28, 2022
1 parent f073529 commit dfb1098
Show file tree
Hide file tree
Showing 10 changed files with 531 additions and 17 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ out
local
test/setup/elasticsearch/elasticsearch-*
vendor
!docker/ironbank/go/src/env2yaml/vendor
.sass-cache
/data
.buildpath
Expand Down
44 changes: 43 additions & 1 deletion docker/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,32 @@ docker_paths:
mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml
mkdir -p $(ARTIFACTS_DIR)/docker/pipeline

public-dockerfiles: public-dockerfiles_oss public_dockerfiles_full public_dockerfiles_ubi8
COPY_IRONBANK_FILES = $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md

$(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml: data/logstash/config/pipelines.yml
$(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml: data/logstash/config/logstash-full.yml
$(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties: data/logstash/config/log4j2.properties
$(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf: data/logstash/pipeline/default.conf
$(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go
$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod: ironbank/go/src/env2yaml/go.mod
$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum: ironbank/go/src/env2yaml/go.sum
$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt: ironbank/go/src/env2yaml/vendor/modules.txt
$(ARTIFACTS_DIR)/ironbank/LICENSE: ironbank/LICENSE
$(ARTIFACTS_DIR)/ironbank/README.md: ironbank/README.md

$(ARTIFACTS_DIR)/ironbank/%:
cp -f $< $@

ironbank_docker_paths:
mkdir -p $(ARTIFACTS_DIR)/ironbank/
mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts
mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/bin
mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/config
mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor
mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline

public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_ubi8 public-dockerfiles_ironbank

public-dockerfiles_full: venv templates/Dockerfile.j2 docker_paths $(COPY_FILES)
jinja2 \
Expand Down Expand Up @@ -128,6 +153,23 @@ public-dockerfiles_ubi8: venv templates/Dockerfile.j2 docker_paths $(COPY_FILES)
cp $(ARTIFACTS_DIR)/Dockerfile-ubi8 Dockerfile && \
tar -zcf ../logstash-ubi8-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline

public-dockerfiles_ironbank: templates/hardening_manifest.yaml templates/Dockerfile.j2 ironbank_docker_paths $(COPY_IRONBANK_FILES)
jinja2 \
-D elastic_version='$(ELASTIC_VERSION)' \
templates/hardening_manifest.yaml > $(ARTIFACTS_DIR)/ironbank/hardening_manifest.yaml && \
jinja2 \
-D created_date='$(BUILD_DATE)' \
-D elastic_version='$(ELASTIC_VERSION)' \
-D arch='${ARCHITECTURE}' \
-D version_tag='$(VERSION_TAG)' \
-D image_flavor='ironbank' \
-D local_artifacts='false' \
-D release='$(RELEASE)' \
templates/Dockerfile.j2 > $(ARTIFACTS_DIR)/Dockerfile-ironbank && \
cd $(ARTIFACTS_DIR)/ironbank && \
cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \
tar -zcf ../logstash-ironbank-$(VERSION_TAG)-docker-build-context.tar.gz scripts Dockerfile hardening_manifest.yaml LICENSE README.md

# Push the image to the dedicated push endpoint at "push.docker.elastic.co"
push:
$(foreach FLAVOR, $(IMAGE_FLAVORS), \
Expand Down
280 changes: 280 additions & 0 deletions docker/ironbank/LICENSE

Large diffs are not rendered by default.

34 changes: 34 additions & 0 deletions docker/ironbank/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Logstash
Logstash is part of the [Elastic Stack](https://www.elastic.co/products) along with Elasticsearch, Kibana, and Beats. Logstash is a server-side data processing pipeline that ingests data from a multitude of sources simultaneously, transforms it, and then sends it to your favorite "stash." (Ours is Elasticsearch, naturally.). Logstash has over 200 plugins, and you can write your own very easily as well.

For more info, see <https://www.elastic.co/products/logstash>

### Installation instructions

Please follow the documentation on [how to install Logstash with Docker](https://www.elastic.co/guide/en/logstash/current/docker.html).

## Documentation and Getting Started

You can find the documentation and getting started guides for Logstash
on the [elastic.co site](https://www.elastic.co/guide/en/logstash/current/getting-started-with-logstash.html).

### Where to file issues and PRs

- [Issues](https://github.com/elastic/logstash/issues)
- [PRs](https://github.com/elastic/logstash/pulls)

**Please open new issues and pull requests for plugins under its own repository**

For example, if you have to report an issue/enhancement for the Elasticsearch output, please do so [here](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues).

## Need Help?

- [Logstash Forum](https://discuss.elastic.co/c/logstash)
- [Logstash Documentation](https://www.elastic.co/guide/en/logstash/current/index.html)
- [Elastic Support](https://www.elastic.co/subscriptions)

## Project Principles

* Community: If a newbie has a bad time, it's a bug.
* Software: Make it work, then make it right, then make it fast.
* Technology: If it doesn't do a thing today, we can make it do it tomorrow.
5 changes: 5 additions & 0 deletions docker/ironbank/go/src/env2yaml/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module env2yaml

go 1.13

require gopkg.in/yaml.v2 v2.3.0
3 changes: 3 additions & 0 deletions docker/ironbank/go/src/env2yaml/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
2 changes: 2 additions & 0 deletions docker/ironbank/go/src/env2yaml/vendor/modules.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# gopkg.in/yaml.v2 v2.3.0
gopkg.in/yaml.v2
94 changes: 81 additions & 13 deletions docker/templates/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -18,35 +18,83 @@
{% set package_manager = 'microdnf' -%}
# Minimal distributions do not ship with en language packs.
{% set locale = 'C.UTF-8' -%}
{% elif image_flavor == 'ironbank' -%}
{% set package_manager = 'yum' -%}
{% else -%}
{% set base_image = 'ubuntu:20.04' -%}
{% set package_manager = 'apt-get' -%}
{% set locale = 'en_US.UTF-8' -%}
{% endif -%}


{% if image_flavor == 'ironbank' -%}
ARG BASE_REGISTRY=registry1.dsop.io
ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8
ARG BASE_TAG=8.6
ARG LOGSTASH_VERSION={{ elastic_version }}
ARG GOLANG_VERSION=1.17.8

FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS env2yaml

ARG GOLANG_VERSION

# install golang
RUN yum update -y && yum install -y git
COPY go${GOLANG_VERSION}.linux-amd64.tar.gz /opt/go.tar.gz
RUN tar -C /usr/local -xzf /opt/go.tar.gz
ENV PATH=$PATH:/usr/local/go/bin

# compile the env2yaml tool
COPY v2.3.0.tar.gz /opt/env2yaml.tar.gz
COPY scripts/go /usr/local/src/go
WORKDIR /usr/local/src/go/src/env2yaml
RUN mkdir -p vendor/gopkg.in
RUN tar -zxf /opt/env2yaml.tar.gz -C vendor/gopkg.in
RUN mv vendor/gopkg.in/yaml-2.3.0 vendor/gopkg.in/yaml.v2
ENV GOPATH=/usr/local/src/go
RUN go build -mod vendor

# stage 1: unpack logstash
FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS builder

ARG LOGSTASH_VERSION

WORKDIR /usr/share/
COPY logstash-${LOGSTASH_VERSION}-linux-x86_64.tar.gz /opt/logstash.tar.gz

RUN tar zxf /opt/logstash.tar.gz && \
mv /usr/share/logstash-${LOGSTASH_VERSION} /usr/share/logstash

FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}

{% else -%}
FROM {{ base_image }}

{% endif -%}

RUN for iter in {1..10}; do \
{% if image_flavor != 'ubi8' -%}
{% if image_flavor == 'full' or image_flavor == 'oss' -%}
export DEBIAN_FRONTEND=noninteractive && \
{% endif -%}
{{ package_manager }} update -y && \
{% if image_flavor != 'ironbank' -%}
{{ package_manager }} upgrade -y && \
{% endif -%}
{{ package_manager }} install -y procps findutils tar gzip curl && \
{% if image_flavor == 'ubi8' -%}
{% if image_flavor == 'ubi8' or image_flavor == 'ironbank' -%}
{{ package_manager }} install -y which shadow-utils && \
{% else -%}
{{ package_manager }} install -y locales && \
{% endif -%}
{{ package_manager }} clean all && \
{% if image_flavor != 'ubi8' -%}
{% if image_flavor == 'full' or image_flavor == 'oss' -%}
locale-gen 'en_US.UTF-8' && \
{{ package_manager }} clean metadata && \
{% endif -%}
exit_code=0 && break || exit_code=$? && \
echo "packaging error: retry $iter in 10s" && \
{{ package_manager }} clean all && \
{% if image_flavor != 'ubi8' -%}
{% if image_flavor == 'full' or image_flavor == 'oss' -%}
{{ package_manager }} clean metadata && \
{% endif -%}
sleep 10; done; \
Expand All @@ -55,13 +103,21 @@ RUN for iter in {1..10}; do \
# Provide a non-root user to run the process.
RUN groupadd --gid 1000 logstash && \
adduser --uid 1000 --gid 1000 \
--home /usr/share/logstash --no-create-home \
{% if image_flavor != 'ironbank' %} --home {% else %} --home-dir {% endif -%} /usr/share/logstash --no-create-home \
logstash

{% if image_flavor == 'ironbank' -%}
WORKDIR /usr/share/logstash
COPY --from=env2yaml /usr/local/src/go/src/env2yaml/env2yaml /usr/local/bin/env2yaml
COPY --from=builder --chown=1000:0 /usr/share/logstash /usr/share/logstash
{% endif -%}

# Add Logstash itself.
RUN curl -Lo - {{ url_root }}/{{ tarball }} | \
RUN \
{% if image_flavor != 'ironbank' -%} curl -Lo - {{ url_root }}/{{ tarball }} | \
tar zxf - -C /usr/share && \
mv /usr/share/logstash-{{ elastic_version }} /usr/share/logstash && \
{% endif -%}
chown --recursive logstash:logstash /usr/share/logstash/ && \
chown -R logstash:root /usr/share/logstash && \
chmod -R g=u /usr/share/logstash && \
Expand All @@ -71,14 +127,15 @@ RUN curl -Lo - {{ url_root }}/{{ tarball }} | \
find /usr/share/logstash -type d -exec chmod g+s {} \; && \
ln -s /usr/share/logstash /opt/logstash


{% if image_flavor != 'ironbank' -%}
WORKDIR /usr/share/logstash

{% endif -%}
ENV ELASTIC_CONTAINER true
ENV PATH=/usr/share/logstash/bin:$PATH

# Provide a minimal configuration, so that simple invocations will provide
# a good experience.
{% if image_flavor != 'ironbank' -%}
ADD config/pipelines.yml config/pipelines.yml
{% if image_flavor == 'oss' -%}
ADD config/logstash-oss.yml config/logstash.yml
Expand All @@ -88,21 +145,28 @@ ADD config/logstash-full.yml config/logstash.yml
ADD config/log4j2.properties config/
ADD pipeline/default.conf pipeline/logstash.conf
RUN chown --recursive logstash:root config/ pipeline/

# Ensure Logstash gets the correct locale by default.
ENV LANG={{ locale }} LC_ALL={{ locale }}

ADD env2yaml/env2yaml /usr/local/bin/
# Place the startup wrapper script.
ADD bin/docker-entrypoint /usr/local/bin/
{% else -%}
COPY scripts/config/pipelines.yml config/pipelines.yml
COPY scripts/config/logstash.yml config/logstash.yml
COPY scripts/config/log4j2.properties config/
COPY scripts/pipeline/default.conf pipeline/logstash.conf
RUN chown --recursive logstash:root config/ pipeline/
# Place the startup wrapper script.
COPY scripts/bin/docker-entrypoint /usr/local/bin/
{% endif -%}

RUN chmod 0755 /usr/local/bin/docker-entrypoint

USER 1000

ADD env2yaml/env2yaml /usr/local/bin/

EXPOSE 9600 5044


{% if image_flavor != 'ironbank' -%}
LABEL org.label-schema.schema-version="1.0" \
org.label-schema.vendor="Elastic" \
org.opencontainers.image.vendor="Elastic" \
Expand All @@ -125,6 +189,10 @@ LABEL org.label-schema.schema-version="1.0" \
vendor="Elastic" \
{% endif -%}
org.opencontainers.image.created={{ created_date }}
{% endif -%}

{% if image_flavor == 'ironbank' -%}
HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9600 || exit 1
{% endif -%}

ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
72 changes: 72 additions & 0 deletions docker/templates/hardening_manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
---
apiVersion: v1

# The repository name in registry1, excluding /ironbank/
name: "elastic/logstash/logstash"

# List of tags to push for the repository in registry1
# The most specific version should be the first tag and will be shown
# on ironbank.dsop.io
tags:
- "{{ elastic_version }}"
- "latest"

# Build args passed to Dockerfile ARGs
args:
BASE_IMAGE: "redhat/ubi/ubi8"
BASE_TAG: "8.6"
LOGSTASH_VERSION: "{{ elastic_version }}"
GOLANG_VERSION: "1.17.8"

# Docker image labels
labels:
org.opencontainers.image.title: "logstash"
## Human-readable description of the software packaged in the image
org.opencontainers.image.description: "Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'"
## License(s) under which contained software is distributed
org.opencontainers.image.licenses: "Elastic License"
## URL to find more information on the image
org.opencontainers.image.url: "https://www.elastic.co/products/logstash"
## Name of the distributing entity, organization or individual
org.opencontainers.image.vendor: "Elastic"
org.opencontainers.image.version: "{{ elastic_version }}"
## Keywords to help with search (ex. "cicd,gitops,golang")
# mil.dso.ironbank.image.keywords: "FIXME"
## This value can be "opensource" or "commercial"
# mil.dso.ironbank.image.type: "FIXME"
## Product the image belongs to for grouping multiple images
mil.dso.ironbank.product.name: "Logstash"

# List of resources to make available to the offline build context
resources:
- filename: logstash-{{ elastic_version }}-linux-x86_64.tar.gz
url: https://artifacts.elastic.co/downloads/logstash/logstash-{{ elastic_version }}-linux-x86_64.tar.gz
validation:
type: sha512
value: <INSERT SHA512 VALUE FROM https://artifacts.elastic.co/downloads/logstash/logstash-{{ elastic_version }}-linux-x86_64.tar.gz.sha512>
- filename: go1.17.8.linux-amd64.tar.gz
url: https://dl.google.com/go/go1.17.8.linux-amd64.tar.gz
validation:
type: sha256
value: 980e65a863377e69fd9b67df9d8395fd8e93858e7a24c9f55803421e453f4f99
- filename: v2.3.0.tar.gz
url: https://github.com/go-yaml/yaml/archive/v2.3.0.tar.gz
validation:
type: sha512
value: ba934e9cb5ebd2346d3897308b71d13bc6471a8dbc0dc0d46a02644ee6b6553d20c20393471b81025b572a9b03e3326bde9c3e8be156474f1a1f91ff027b6a4f

# List of project maintainers
maintainers:
- name: "Nassim Kammah"
username: "nkammah"
email: "nassim.kammah@elastic.co"
- name: "Joao Duarte"
username: "joaodiasduarte"
email: "joao@elastic.co"
- name: "Rob Bavey"
username: "robbavey"
email: "rob.bavey@elastic.co"
- name: "Kaise Cheng"
username: "kaisecheng"
email: "kaise.cheng@elastic.co"

Loading

0 comments on commit dfb1098

Please sign in to comment.