Migrate x-pack-logstash source to logstash

Dockerfile

@@ -8,7 +8,8 @@ ADD buildSrc /opt/logstash/buildSrc
 RUN /opt/logstash/gradlew wrapper
 
 ADD versions.yml /opt/logstash/versions.yml
-ADD LICENSE /opt/logstash/LICENSE
+ADD LICENSE.txt /opt/logstash/LICENSE.txt
+ADD licenses /opt/logstash/licenses
 ADD CONTRIBUTORS /opt/logstash/CONTRIBUTORS
 ADD Gemfile.template /opt/logstash/Gemfile.template
 ADD Rakefile /opt/logstash/Rakefile
@@ -24,6 +25,7 @@ ADD logstash-core /opt/logstash/logstash-core
 ADD logstash-core-plugin-api /opt/logstash/logstash-core-plugin-api
 ADD bin /opt/logstash/bin
 ADD modules /opt/logstash/modules
+ADD x-pack /opt/logstash/x-pack
 ADD ci /opt/logstash/ci
 ADD settings.gradle /opt/logstash/settings.gradle
 

LICENSE.txt

@@ -0,0 +1,13 @@
+Source code in this repository is variously licensed under the Apache License
+Version 2.0, an Apache compatible license, or the Elastic License. Outside of
+the "x-pack" folder, source code in a given file is licensed under the Apache
+License Version 2.0, unless otherwise noted at the beginning of the file or a
+LICENSE file present in the directory subtree declares a separate license.
+Within the "x-pack" folder, source code in a given file is licensed under the
+Elastic License, unless otherwise noted at the beginning of the file or a
+LICENSE file present in the directory subtree declares a separate license.
+
+The build produces two sets of binaries - one set that falls under the Elastic
+License and another set that falls under Apache License Version 2.0. The
+binaries that contain `-oss` in the artifact name are licensed under the Apache
+License Version 2.0.

README.md

@@ -18,14 +18,24 @@ supported platforms, from [downloads page](https://www.elastic.co/downloads/logs
 
 ### Snapshot Builds
 
-For the daring, snapshot builds are available. These builds are created nightly and have undergone no formal QA, so they should **never** be run in production.
-
-| artifact |
-| --- |
-| [tar](https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.tar.gz) |
-| [zip](https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.zip) |
-| [deb](https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.deb) |
-| [rpm](https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.rpm) |
+For the daring, snapshot builds are available.
+These builds are created nightly and have undergone no formal QA, so they should **never** be run in production.
+
+| Complete, with X-Pack | Apache 2.0 licensed    |
+| --------------------- | ---------------------- |
+| [tar-complete][]      | [tar-oss][]            |
+| [zip-complete][]      | [zip-oss][]            |
+| [deb-complete][]      | [deb-oss][]            |
+| [rpm-complete][]      | [rpm-oss][]            |
+
+[tar-complete]: https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.tar.gz
+[zip-complete]: https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.zip
+[deb-complete]: https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.deb
+[rpm-complete]: https://snapshots.elastic.co/downloads/logstash/logstash-7.0.0-alpha1-SNAPSHOT.rpm
+[tar-oss]: https://snapshots.elastic.co/downloads/logstash/logstash-oss-7.0.0-alpha1-SNAPSHOT.tar.gz
+[zip-oss]: https://snapshots.elastic.co/downloads/logstash/logstash-oss-7.0.0-alpha1-SNAPSHOT.zip
+[deb-oss]: https://snapshots.elastic.co/downloads/logstash/logstash-oss-7.0.0-alpha1-SNAPSHOT.deb
+[rpm-oss]: https://snapshots.elastic.co/downloads/logstash/logstash-oss-7.0.0-alpha1-SNAPSHOT.rpm
 
 ## Need Help?
 
@@ -80,6 +90,12 @@ The printed version should be the same as in the `.ruby-version` file.
 
 ### Building Logstash
 
+The Logstash project includes the source code for all of Logstash, including the Elastic-Licensed X-Pack features and functions; to run Logstash from source using only the OSS-licensed code, export the `OSS` environment variable with a value of `true`:
+
+``` sh
+export OSS=true
+```
+
 * To run Logstash from the repo you must first bootstrap the environment:
 
 ```sh
@@ -189,14 +205,21 @@ Note that if a plugin is installed using the plugin manager `bin/logstash-plugin
 
 ## Building Artifacts
 
+Built artifacts will be placed in the `LS_HOME/build` directory, and will create the directory if it is not already present.
+
 You can build a Logstash snapshot package as tarball or zip file
 
 ```sh
 ./gradlew assembleTarDistribution
 ./gradlew assembleZipDistribution
 ```
 
-This will create the artifact `LS_HOME/build` directory
+OSS-only artifacts can similarly be built with their own gradle tasks:
+```sh
+./gradlew assembleOssTarDistribution
+./gradlew assembleOssZipDistribution
+
+```
 
 You can also build .rpm and .deb, but the [fpm](https://github.com/jordansissel/fpm) tool is required.
 
@@ -205,6 +228,13 @@ rake artifact:rpm
 rake artifact:deb
 ```
 
+and:
+
+```sh
+rake artifact:rpm_oss
+rake artifact:deb_oss
+```
+
 ## Project Principles
 
 * Community: If a newbie has a bad time, it's a bug.

build.gradle

@@ -281,7 +281,8 @@ bootstrap.dependsOn installTestGems
 runIntegrationTests.shouldRunAfter tasks.getByPath(":logstash-core:test")
 check.dependsOn runIntegrationTests
 
-String elasticsearchSnapshotURL = "https://snapshots.elastic.co/downloads/elasticsearch/elasticsearch-${version}-SNAPSHOT.tar.gz"
+
+String elasticsearchSnapshotURL = System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: "https://snapshots.elastic.co/downloads/elasticsearch/elasticsearch-${version}-SNAPSHOT.tar.gz"
 String elasticsearchDownloadLocation = "${projectDir}/build/elasticsearch-${version}-SNAPSHOT.tar.gz"
 
 task downloadEs(type: Download) {

ci/acceptance_tests.sh

@@ -6,6 +6,7 @@ set -e
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
 export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
+export OSS=true
 
 SELECTED_TEST_SUITE=$1
 

ci/docker_run.sh

@@ -14,6 +14,10 @@ else
     IMAGE_NAME=$branch_specifier"-"$(date +%s%N)
 fi
 
+if [ "$OSS" == "true" ]; then
+  DOCKER_ENV_OPTS="${DOCKER_ENV_OPTS} --env OSS=true"
+fi
+
 echo "Running Docker CI build for '$IMAGE_NAME' "
 
 # Remove old docker cid just in case
@@ -23,14 +27,19 @@ docker build -t $IMAGE_NAME .
 exit_code=$?; [[ $exit_code != 0 ]] && exit $exit_code
 
 cleanup() {
-  cat docker_cid | xargs docker rm --force -v 
+  if [ -e docker_cid ]; then
+    cat docker_cid | xargs docker rm --force -v 
+  fi
 }
 trap cleanup EXIT
 
 # Run the command, skip the first argument, which is the image name
-echo "Running tests in built docker image"
-docker run --sig-proxy=true --cidfile=docker_cid --rm $IMAGE_NAME ${@:2}
+docker run $DOCKER_ENV_OPTS --cidfile=docker_cid --sig-proxy=true --rm $IMAGE_NAME ${@:2}
+
+# Remove the container cid since we ran cleanly, no need to force rm it if we got to this point
+rm docker_cid
+
 exit_code=$?
 [[ $REMOVE_IMAGE == "true" ]] && docker rmi $IMAGE_NAME
 echo "exiting with code: '$exit_code'"
-exit $exit_code #preserve the exit code from the test run
+exit $exit_code #preserve the exit code from the test run

ci/integration_tests.sh

@@ -9,6 +9,7 @@ export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=
 
 export SPEC_OPTS="--order rand --format documentation"
 export CI=true
+export OSS=true
 
 if [[ $1 = "setup" ]]; then
  echo "Setup only, no tests will be run"

ci/unit_tests.sh

@@ -9,6 +9,7 @@ export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=
 
 export SPEC_OPTS="--order rand --format documentation"
 export CI=true
+export OSS=true
 
 SELECTED_TEST_SUITE=$1
 

config/logstash.yml

@@ -211,4 +211,36 @@
 #
 # Where to find custom plugins
 # path.plugins: []
-
+#
+# ------------ X-Pack Settings (not applicable for OSS build)--------------
+#
+# X-Pack Monitoring
+# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html
+#xpack.monitoring.enabled: false
+#xpack.monitoring.elasticsearch.username: logstash_system
+#xpack.monitoring.elasticsearch.password: password
+#xpack.monitoring.elasticsearch.url: ["https://es1:9200", "https://es2:9200"]
+#xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ]
+#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file
+#xpack.monitoring.elasticsearch.ssl.truststore.password: password
+#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file
+#xpack.monitoring.elasticsearch.ssl.keystore.password: password
+#xpack.monitoring.elasticsearch.ssl.verification_mode: certificate
+#xpack.monitoring.elasticsearch.sniffing: false
+#xpack.monitoring.collection.interval: 10s
+#xpack.monitoring.collection.pipeline.details.enabled: true
+#
+# X-Pack Management
+# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html
+#xpack.management.enabled: false
+#xpack.management.pipeline.id: ["main", "apache_logs"]
+#xpack.management.elasticsearch.username: logstash_admin_user
+#xpack.management.elasticsearch.password: password
+#xpack.management.elasticsearch.url: ["https://es1:9200", "https://es2:9200"]
+#xpack.management.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ]
+#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file
+#xpack.management.elasticsearch.ssl.truststore.password: password
+#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file
+#xpack.management.elasticsearch.ssl.keystore.password: password
+#xpack.management.elasticsearch.sniffing: false
+#xpack.management.logstash.poll_interval: 5s

docs/index.x.asciidoc

@@ -3,7 +3,7 @@
 
 :include-xpack:     true
 :lang:              en
-:xls-repo-dir:      {docdir}/../../logstash-extra/x-pack-logstash/docs/{lang}
+:xls-repo-dir:      {docdir}/../x-pack/docs/{lang}
 :log-repo-dir:      {docdir}
 :plugins-repo-dir:  {docdir}/../../logstash-docs/docs