diff --git a/minnie-kenny.gitconfig b/minnie-kenny.gitconfig new file mode 100644 index 00000000000..1bfec70e27c --- /dev/null +++ b/minnie-kenny.gitconfig @@ -0,0 +1,15 @@ +[secrets] + providers = git secrets --aws-provider + patterns = (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16} + patterns = (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\\s*(:|=>|=)\\s*(\"|')?[A-Za-z0-9/\\+=]{40}(\"|')? + patterns = (\"|')?(AWS|aws|Aws)?_?(ACCOUNT|account|Account)_?(ID|id|Id)?(\"|')?\\s*(:|=>|=)\\s*(\"|')?[0-9]{4}\\-?[0-9]{4}\\-?[0-9]{4}(\"|')? + allowed = AKIAIOSFODNN7EXAMPLE + allowed = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + patterns = private_key + patterns = private_key_id + allowed = \"private_key_id\": \"OMITTED\" + allowed = \"private_key\": \"-----BEGIN PRIVATE KEY-----\\\\nBASE64 ENCODED KEY WITH \\\\n TO REPRESENT NEWLINES\\\\n-----END PRIVATE KEY-----\\\\n\" + allowed = \"client_id\": \"22377410244549202395\" + allowed = `private_key` portion needs + allowed = .Data.private_key + allowed = .Data.service_account.private_key diff --git a/minnie-kenny.sh b/minnie-kenny.sh new file mode 100755 index 00000000000..6a58f91234b --- /dev/null +++ b/minnie-kenny.sh @@ -0,0 +1,199 @@ +#!/bin/sh +# Use this script to ensure git-secrets are setup +# https://minnie-kenny.readthedocs.io/ + +set -eu # -o pipefail isn't supported by POSIX + +minnie_kenny_command_name=${0##*/} +minnie_kenny_quiet=0 +minnie_kenny_strict=0 +minnie_kenny_modify=0 +minnie_kenny_gitconfig="minnie-kenny.gitconfig" + +usage() { + if [ ${minnie_kenny_quiet} -ne 1 ]; then + cat <&2 +Usage: + ${minnie_kenny_command_name} + -f | --force Modify the git config to run git secrets + -n | --no-force Do not modify the git config, only verify installation + -s | --strict Require git-secrets to be setup or fail + -q | --quiet Do not output any status messages + -i | --include=FILE Path to the include for git-config (default: "minnie-kenny.gitconfig") +USAGE + fi + exit 1 +} + +run_command() { if [ ${minnie_kenny_quiet} -ne 1 ]; then "$@"; else "$@" >/dev/null 2>&1; fi; } +echo_out() { if [ ${minnie_kenny_quiet} -ne 1 ]; then echo "$@"; fi; } +echo_err() { if [ ${minnie_kenny_quiet} -ne 1 ]; then echo "$@" 1>&2; fi; } + +process_arguments() { + while [ $# -gt 0 ]; do + case "$1" in + -q | --quiet) + minnie_kenny_quiet=1 + shift 1 + ;; + -s | --strict) + minnie_kenny_strict=1 + shift 1 + ;; + -f | --force) + minnie_kenny_modify=1 + shift 1 + ;; + -n | --no-force) + minnie_kenny_modify=0 + shift 1 + ;; + -i) + shift 1 + minnie_kenny_gitconfig="${1:-}" + if [ "${minnie_kenny_gitconfig}" = "" ]; then break; fi + shift 1 + ;; + --include=*) + minnie_kenny_gitconfig="${1#*=}" + shift 1 + ;; + --help) + usage + ;; + *) + echo_err "Unknown argument: $1" + usage + ;; + esac + done + + if [ "${minnie_kenny_gitconfig}" = "" ]; then + echo_err "Error: you need to provide a git-config include file." + usage + fi +} + +# Exits if this system or directory is not setup to run git / minnie-kenny.sh / git-secrets +validate_setup() { + if ! command -v git >/dev/null 2>&1; then + if [ ${minnie_kenny_strict} -eq 0 ]; then + echo_out "\`git\` not found. Not checking for git-secrets." + exit 0 + else + echo_err "Error: \`git\` not found." + exit 1 + fi + fi + + minnie_kenny_is_work_tree="$(git rev-parse --is-inside-work-tree 2>/dev/null || echo false)" + + if [ "${minnie_kenny_is_work_tree}" != "true" ]; then + if [ ${minnie_kenny_strict} -eq 0 ]; then + echo_out "Not a git working tree. Not checking for git-secrets." + exit 0 + else + echo_err "Error: Not a git working tree." + exit 1 + fi + fi + + minnie_kenny_git_dir="$(git rev-parse --absolute-git-dir)" + if [ ! -f "${minnie_kenny_git_dir}/../${minnie_kenny_gitconfig}" ]; then + echo_err "Error: ${minnie_kenny_gitconfig} was not found next to the directory ${minnie_kenny_git_dir}" + exit 1 + fi + + if ! command -v git-secrets >/dev/null 2>&1; then + echo_err "\`git-secrets\` was not found while \`git\` was found." \ + "\`git-secrets\` must be installed first before using ${minnie_kenny_command_name}." \ + "See https://github.com/awslabs/git-secrets#installing-git-secrets" + exit 1 + fi +} + +# Echo 1 if the hook contains a line that starts with "git secrets" otherwise echo 0 +check_hook() { + path="${minnie_kenny_git_dir}/hooks/$1" + if grep -q "^git secrets " "${path}" 2>/dev/null; then + echo 1 + else + echo 0 + fi +} + +# Ensures git secrets hooks are installed along with the configuration to read in the minnie-kenny.gitconfig +check_installation() { + expected_hooks=0 + actual_hooks=0 + + for path in "commit-msg" "pre-commit" "prepare-commit-msg"; do + increment=$(check_hook ${path}) + actual_hooks=$((actual_hooks + increment)) + expected_hooks=$((expected_hooks + 1)) + done + + if [ 0 -lt ${actual_hooks} ] && [ ${actual_hooks} -lt ${expected_hooks} ]; then + # Only some of the hooks are setup, meaning someone updated the hook files in an unexpected way. + # Warn and exit as we cannot fix this with a simple `git secrets --install`. + echo_err "Error: git-secrets is not installed into all of the expected git hooks." \ + "Double check the 'commit-msg' 'pre-commit' and 'prepare-commit-msg' under the directory" \ + "${minnie_kenny_git_dir}/hooks and consider running \`git secrets --install --force\`." + exit 1 + fi + + # Begin checking for fixable errors + found_fixable_errors=0 + + if [ ${actual_hooks} -eq 0 ]; then + if [ ${minnie_kenny_modify} -eq 1 ]; then + run_command git secrets --install + else + echo_err "Error: git-secrets is not installed into the expected git hooks" \ + "'commit-msg' 'pre-commit' and 'prepare-commit-msg'." + found_fixable_errors=1 + fi + fi + + # Allow the minnie-kenny.gitconfig in `git secrets --scan` + if ! git config --get-all secrets.allowed | grep -Fxq "^${minnie_kenny_gitconfig}:[0-9]+:"; then + if [ ${minnie_kenny_modify} -eq 1 ]; then + run_command git config --add secrets.allowed "^${minnie_kenny_gitconfig}:[0-9]+:" + else + echo_err "Error: The expression '^${minnie_kenny_gitconfig}:[0-9]+:' should be allowed by git secrets." + found_fixable_errors=1 + fi + fi + + # Allow minnie-kenny.gitconfig to appear in `git secrets --scan-history` + if ! git config --get-all secrets.allowed | grep -Fxq "^[0-9a-f]+:${minnie_kenny_gitconfig}:[0-9]+:"; then + if [ ${minnie_kenny_modify} -eq 1 ]; then + run_command git config --add secrets.allowed "^[0-9a-f]+:${minnie_kenny_gitconfig}:[0-9]+:" + else + echo_err "Error: The expression '^[0-9a-f]+:${minnie_kenny_gitconfig}:[0-9]+:' should be allowed by git secrets." + found_fixable_errors=1 + fi + fi + + if ! git config --get-all include.path | grep -Fxq "../${minnie_kenny_gitconfig}"; then + if [ ${minnie_kenny_modify} -eq 1 ]; then + run_command git config --add include.path "../${minnie_kenny_gitconfig}" + else + echo_err "Error: The path '../${minnie_kenny_gitconfig}' should be an included path in the git config." + found_fixable_errors=1 + fi + fi + + if [ ${found_fixable_errors} -ne 0 ]; then + echo_err "Error: The above errors may be fixed by re-running ${minnie_kenny_command_name} with -f / --force." + exit 1 + fi +} + +main() { + process_arguments "$@" + validate_setup + check_installation +} + +main "$@" diff --git a/project/ContinuousIntegration.scala b/project/ContinuousIntegration.scala index ec6a3107a42..4ed05a3cc94 100644 --- a/project/ContinuousIntegration.scala +++ b/project/ContinuousIntegration.scala @@ -1,3 +1,4 @@ +import Testing._ import sbt.Keys._ import sbt._ import sbt.io.Path._ @@ -20,6 +21,7 @@ object ContinuousIntegration { IO.copyDirectory(srcCiResources.value, targetCiResources.value) }, renderCiResources := { + minnieKenny.toTask("").value copyCiResources.value val log = streams.value.log if (!vaultToken.value.exists()) { diff --git a/project/Testing.scala b/project/Testing.scala index bf7d4c681c0..ad3592a26f9 100644 --- a/project/Testing.scala +++ b/project/Testing.scala @@ -2,6 +2,10 @@ import Dependencies._ import sbt.Defaults._ import sbt.Keys._ import sbt._ +import complete.DefaultParsers._ +import sbt.util.Logger + +import scala.sys.process._ object Testing { private val AllTests = config("alltests") extend Test @@ -21,6 +25,8 @@ object Testing { DbmsTestTag ) + val minnieKenny = inputKey[Unit]("Run minnie-kenny.") + private val excludeTestTags: Seq[String] = sys.env .get("CROMWELL_SBT_TEST_EXCLUDE_TAGS") @@ -52,6 +58,30 @@ object Testing { "300", ) + /** Run minnie-kenny only once per sbt invocation. */ + class MinnieKennySingleRunner() { + private val mutex = new Object + private var resultOption: Option[Int] = None + + /** Run using the logger, throwing an exception only on the first failure. */ + def runOnce(log: Logger, args: Seq[String]): Unit = { + mutex synchronized { + if (resultOption.isEmpty) { + log.debug(s"Running minnie-kenny.sh${args.mkString(" ", " ", "")}") + val result = ("./minnie-kenny.sh" +: args) ! log + resultOption = Option(result) + if (result == 0) + log.debug("Successfully ran minnie-kenny.sh") + else + sys.error("Running minnie-kenny.sh failed. Please double check for errors above.") + } + } + } + } + + // Only run one minnie-kenny.sh at a time! + private lazy val minnieKennySingleRunner = new MinnieKennySingleRunner + val testSettings = List( libraryDependencies ++= testDependencies.map(_ % Test), // `test` (or `assembly`) - Run all tests, except docker and integration and DBMS @@ -61,7 +91,17 @@ object Testing { // Add scalameter as a test framework in the CromwellBenchmarkTest scope testFrameworks in CromwellBenchmarkTest += new TestFramework("org.scalameter.ScalaMeterFramework"), // Don't execute benchmarks in parallel - parallelExecution in CromwellBenchmarkTest := false + parallelExecution in CromwellBenchmarkTest := false, + // Make sure no secrets are commited to git + minnieKenny := { + val log = streams.value.log + val args = spaceDelimited("").parsed + minnieKennySingleRunner.runOnce(log, args) + }, + test in Test := { + minnieKenny.toTask("").value + (test in Test).value + }, ) val integrationTestSettings = List( diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 39a645bbaf2..5430206344d 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -68,6 +68,8 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_RESOURCES_SOURCES="${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/resources" CROMWELL_BUILD_RESOURCES_DIRECTORY="${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/resources" + CROMWELL_BUILD_GIT_SECRETS_DIRECTORY="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/git-secrets" + CROMWELL_BUILD_GIT_SECRETS_COMMIT="ad82d68ee924906a0401dfd48de5057731a9bc84" CROMWELL_BUILD_WAIT_FOR_IT_FILENAME="wait-for-it.sh" CROMWELL_BUILD_WAIT_FOR_IT_BRANCH="db049716e42767d39961e95dd9696103dca813f1" CROMWELL_BUILD_WAIT_FOR_IT_URL="https://raw.githubusercontent.com/vishnubob/wait-for-it/${CROMWELL_BUILD_WAIT_FOR_IT_BRANCH}/${CROMWELL_BUILD_WAIT_FOR_IT_FILENAME}" @@ -214,6 +216,8 @@ cromwell::private::create_build_variables() { export CROMWELL_BUILD_EXIT_FUNCTIONS export CROMWELL_BUILD_GENERATE_COVERAGE export CROMWELL_BUILD_GIT_HASH_SUFFIX + export CROMWELL_BUILD_GIT_SECRETS_COMMIT + export CROMWELL_BUILD_GIT_SECRETS_DIRECTORY export CROMWELL_BUILD_GIT_USER_EMAIL export CROMWELL_BUILD_GIT_USER_NAME export CROMWELL_BUILD_HEARTBEAT_MINUTES @@ -633,6 +637,26 @@ cromwell::private::install_wait_for_it() { chmod +x "$CROMWELL_BUILD_WAIT_FOR_IT_SCRIPT" } +cromwell::private::install_git_secrets() { + # Only install git-secrets on CI. Users should have already installed the executable. + if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]]; then + git clone https://github.com/awslabs/git-secrets.git "${CROMWELL_BUILD_GIT_SECRETS_DIRECTORY}" + pushd "${CROMWELL_BUILD_GIT_SECRETS_DIRECTORY}" > /dev/null + git checkout "${CROMWELL_BUILD_GIT_SECRETS_COMMIT}" + export PATH="${PATH}:${PWD}" + popd > /dev/null + fi +} + +cromwell::private::install_minnie_kenny() { + # Only install minnie-kenny on CI. Users should have already run the script themselves. + if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]]; then + pushd "${CROMWELL_BUILD_ROOT_DIRECTORY}" > /dev/null + ./minnie-kenny.sh --force + popd > /dev/null + fi +} + cromwell::private::start_docker() { local docker_image local docker_cid_file @@ -1097,6 +1121,8 @@ cromwell::build::setup_common_environment() { cromwell::private::verify_secure_build cromwell::private::verify_pull_request_build cromwell::private::make_build_directories + cromwell::private::install_git_secrets + cromwell::private::install_minnie_kenny cromwell::private::setup_secure_resources case "${CROMWELL_BUILD_PROVIDER}" in diff --git a/src/ci/bin/testCheckPublish.sh b/src/ci/bin/testCheckPublish.sh index c640215bd63..4118509e18b 100755 --- a/src/ci/bin/testCheckPublish.sh +++ b/src/ci/bin/testCheckPublish.sh @@ -11,3 +11,5 @@ cromwell::build::pip_install mkdocs mkdocs build -s sbt checkRestApiDocs +package assembly dockerPushCheck +doc + +git secrets --scan-history diff --git a/src/ci/bin/testSbt.sh b/src/ci/bin/testSbt.sh index 36f77d5fa95..0edfb9f2306 100755 --- a/src/ci/bin/testSbt.sh +++ b/src/ci/bin/testSbt.sh @@ -27,7 +27,10 @@ esac export CROMWELL_SBT_TEST_EXCLUDE_TAGS export CROMWELL_SBT_TEST_SPAN_SCALE_FACTOR -sbt -Dakka.test.timefactor=${CROMWELL_SBT_TEST_SPAN_SCALE_FACTOR} -Dbackend.providers.Local.config.filesystems.local.localization.0=copy coverage test +sbt \ + -Dakka.test.timefactor=${CROMWELL_SBT_TEST_SPAN_SCALE_FACTOR} \ + -Dbackend.providers.Local.config.filesystems.local.localization.0=copy \ + coverage test cromwell::build::generate_code_coverage