diff --git a/.versionrc.json b/.versionrc.json new file mode 120000 index 0000000..f0385a9 --- /dev/null +++ b/.versionrc.json @@ -0,0 +1 @@ +etc/.versionrc.json \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..52564cf --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,106 @@ +# Changelog + +All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. + +## 0.1.0 (2022-10-25) + + +### Features + +* Add --env flag to cbuild +* Add arg processing to cbuild script +* Add aws-cli helper script +* Add bazel-debian helper script +* Add builders/utils docker image +* Add hadolint to lint Dockerfiles +* Add optional flags to cbuild tool +* Add preliminary support for commit-and-tag-version and copybara +* Add the release-please tool +* Add toolchain short hash to bazel output_user_root path +* Add tools/lib/builder.sh +* **build:** Add GitHub CLI tool https://cli.github.com/ +* Determine workspace mount point from docker inspect if inside docker container +* Inject clang-version as a bazel action_env +* Migrate generation of builders/release container image to Dockerfile +* Move image directories to images/ top-level directory +* Overhaul building on amazonlinux2 +* Remove python build dependencies from bazel +* Set BUILD_ARCH env var in docker images +* Update release image to node v18 +* Upgrade to bazel 5.3.2 +* Upgrade to clang v14 on bazel-debian +* Use Packer to build AMI. + + +### Bug Fixes + +* Add builders/tools/normalize-dist to chmod/chgrp/chown dist/ directory tree +* Add get_workspace_mount function to encapsulate code block +* Add python version to action_env +* Add/remove basic pre-commit hooks +* Adopt shellcheck +* Avoid installing recommended debian packages +* Avoid use of git rev-parse to determine tools path +* Bump to latest version of bazelisk +* Clean bazel_root for smaller docker image +* Correct argument handling in cbuild script +* Correct errors generating Amazon Linux 2-based builder image +* Define python3 toolchain +* Drop packer from build-debian image +* Ensure /etc/gitconfig is readable by all +* Improve cbuild help text +* Install bazel version as specified in .bazelversion +* Invoke addlicense for all text files +* Modifications as indicated by shellcheck +* Mount $HOME/aws in aws-cli container +* Move bazel env vars from comments to help text +* Move builder-related configs to builders/etc +* Move WORKSPACE definition to cbuild script global +* Only propagate AWS env vars into amazonlinux2 build container +* Pin version of bazelisk +* Pin version of libc++-dev +* Pin version of python3.8 +* Print pre-commit version rather than help +* Remove container when get-architecture exits +* Remove debugging statement +* Remove dockerfile linter ignore and correct ENTRYPOINT +* Remove pre-commit config from build-debian +* Remove shellcheck from build-debian +* Remove unused nitro_enclave_image bazel rule +* Set bazel output_base to accommodate distinct workspaces +* Set bazel output_user_root in image bazelrc +* Set locale in build-debian +* Set WORKSPACE correctly from submodule +* Set WORKSPACE variable +* Switch from hardcoded arch to using dpkg --print-architecture +* Update normalize-dist to function inside build container +* Update pre-commit to use cbuild +* Use PRE_COMMIT_TOOL env var +* Various path-related fixes + + +### Build System + +* Add arch to docker image tags +* Add get_builder_image_tagged tool to determine a content-based tag +* Add get-architecture helper script +* Add missing imports into nitro BUILD +* Add tools/pre-commit +* Correct propagation of quoted args in gh wrapper +* Move commit-and-tag-version into tools dir +* Move gh into tools dir +* Optionally build the AMI +* Propagate status code in exit functions +* Reduce redundant installation commands +* Remove release-please tool +* Rename builders/bazel to build-debian +* Simplify use of npm image in container_run_and_commit() +* Support GH_TOKEN env var + + +### Documentation + +* Add top-level README.md +* Improve prose in README.md +* Move docker build instructions to README.md +* Reformat and lint markdown diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..20de409 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# Privacy Sandbox Builders + +Build tools and docker images used by the [Privacy Sandbox](https://github.com/privacysandbox) +open-source ecosystem. + +## Building Docker Images + +To build a docker image directly, you can use the `tools/get-builder-image-tagged` tool. + +Alternatively, you can use `docker buildx build` to create the image using a +command like this: + +```sh +tar --create --dereference --gzip --directory=images/build-debian . | \ + docker buildx build - --tag privacysandbox/builders/build-debian:latest` +``` + diff --git a/etc/.bazelversion b/etc/.bazelversion new file mode 100644 index 0000000..84197c8 --- /dev/null +++ b/etc/.bazelversion @@ -0,0 +1 @@ +5.3.2 diff --git a/etc/.clang-format b/etc/.clang-format new file mode 100644 index 0000000..8c45f0c --- /dev/null +++ b/etc/.clang-format @@ -0,0 +1,33 @@ +BasedOnStyle: Google +Standard: c++17 +DerivePointerAlignment: false +SortIncludes: true +IncludeBlocks: Regroup +IncludeCategories: + # gtest, this should be put first in tests. + - Regex: '^ from OS. + - Regex: '^<[_A-Za-z0-9-]+\.h>' + Priority: 30 + # 2nd level .h headers in <>, POSIX standard. + - Regex: '^<(sys|arpa|net|netinet)\/[_A-Za-z0-9-]+\.h>' + Priority: 30 + # Linux-specific .h headers in <>. + - Regex: '^' + Priority: 40 + # Headers in <> without extension, these are basically C++ STL headers + - Regex: '^<[\/_A-Za-z0-9-]+>' + Priority: 50 + # Headers in <> from specific external libraries. + - Regex: '^<(grpcpp|absl)\/' + Priority: 60 + # Any other uncaught headers in <> + - Regex: '^<' + Priority: 70 + # Headers in "" of current directory this should be the last category. + - Regex: '^"[_A-Za-z0-9-]+\.h' + Priority: 200 + # Headers in "" with directory hierarchy + - Regex: '^"[\/_A-Za-z0-9-]+' + Priority: 80 diff --git a/etc/.hadolint.yaml b/etc/.hadolint.yaml new file mode 100644 index 0000000..73dc912 --- /dev/null +++ b/etc/.hadolint.yaml @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://github.com/hadolint/hadolint for more information + +failure-threshold: error + +trustedRegistries: +- docker.io diff --git a/etc/.markdownlint-cli2.yaml b/etc/.markdownlint-cli2.yaml new file mode 100644 index 0000000..0671ea1 --- /dev/null +++ b/etc/.markdownlint-cli2.yaml @@ -0,0 +1,43 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +config: + line-length: + line_length: 120 + #stern: true + code_blocks: false + + # these are apparently in conflict with prettier's markdown formatting + list-marker-space: false + list-indent: false + ul-indent: false + + headings: false + + proper-names: + code_blocks: false + names: + - CommonMark + - JavaScript + - Markdown + - markdown-it + - markdownlint + - markdownlint-cli2 + - Node.js + +fix: true + +ignores: +- CHANGELOG.md +- google_internal/LATEST_RELEASE.md diff --git a/etc/.pre-commit-config.yaml b/etc/.pre-commit-config.yaml new file mode 100644 index 0000000..deced51 --- /dev/null +++ b/etc/.pre-commit-config.yaml @@ -0,0 +1,148 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +exclude: (?x)^( + bazel-(bin|out|testlogs|workspace)/.*| + .bazel_output/.*| + version.txt + )$ + +fail_fast: true +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: mixed-line-ending + - id: trailing-whitespace + - id: check-case-conflict + - id: check-merge-conflict + - id: check-yaml + - id: check-json + - id: check-symlinks + - id: check-added-large-files + - id: check-vcs-permalinks + - id: check-executables-have-shebangs + - id: detect-private-key + +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: git-check + - id: script-must-not-have-extension + exclude: '^google_internal/.*/kokoro_(presubmit|continuous).sh$' + - id: script-must-have-extension + - id: require-ascii + - id: shellcheck + exclude: '^(production|tools|google_internal|builders/images)/.*$' + +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: v14.0.6 + hooks: + - id: clang-format + types_or: + - c++ + - c + +- repo: local + hooks: + - id: addlicense + name: addlicense + language: golang + additional_dependencies: + - github.com/google/addlicense@v1.0.0 + always_run: false + pass_filenames: true + entry: addlicense -v + types_or: + - text + + - id: addlicense-check + name: addlicense check + language: golang + additional_dependencies: + - github.com/google/addlicense@v1.0.0 + always_run: false + pass_filenames: true + entry: addlicense -check + types_or: + - text + + - id: terraform-fmt + name: terraform fmt + language: script + pass_filenames: false + entry: tools/terraform fmt -write=true -recursive + types_or: + - terraform + + - id: hadolint-docker + name: Lint Dockerfiles + description: Runs hadolint via docker image to lint Dockerfiles + language: script + types_or: + - dockerfile + entry: builders/tools/hadolint + +- repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + name: prettier markdown + types_or: + - markdown + +- repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.5.1 + hooks: + - id: markdownlint-cli2 + name: lint markdown + +- repo: local + hooks: + - id: buildifier + name: buildifier + description: Format bazel WORKSPACE, BUILD and .bzl files with a standard convention. + language: golang + additional_dependencies: + - github.com/bazelbuild/buildtools/buildifier@5.1.0 + always_run: true + pass_filenames: true + types_or: + - bazel + entry: buildifier + args: + - -lint=fix + - -mode=fix + - -warnings=all + +- repo: https://github.com/cpplint/cpplint + rev: 1.6.1 + hooks: + - id: cpplint + types_or: + - c++ + - c + args: + - --filter=-build/c++11,+build/c++17,-build/header_guard,-build/include_order,+build/include_what_you_use,-build/include_subdir,-readability/casting,-readability/todo,-runtime/references + - --quiet + +- repo: https://github.com/psf/black + rev: 22.8.0 + hooks: + - id: black + name: black python formatter diff --git a/etc/.prettierrc.yaml b/etc/.prettierrc.yaml new file mode 100644 index 0000000..bf26ea6 --- /dev/null +++ b/etc/.prettierrc.yaml @@ -0,0 +1,25 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +printWidth: 120 +trailingComma: es5 +tabWidth: 2 +useTabs: false +singleQuote: true +proseWrap: always +overrides: +- files: '*.md' + options: + tabWidth: 4 + printWidth: 100 diff --git a/etc/.versionrc.json b/etc/.versionrc.json new file mode 100644 index 0000000..41be1f9 --- /dev/null +++ b/etc/.versionrc.json @@ -0,0 +1,27 @@ +{ + "types": [ + {"type": "feat", "section": "Features"}, + {"type": "fix", "section": "Bug Fixes"}, + {"type": "build", "section": "Build System"}, + {"type": "docs", "section": "Documentation"}, + {"type": "internal", "hidden": true}, + {"type": "chore", "hidden": true}, + {"type": "test", "hidden": true}, + {"type": "refactor", "hidden": true} + ], + "commitUrlFormat": " ", + "issueUrlFormat": " ", + "tagPrefix": "release-", + "packageFiles": [ + { + "filename": "version.txt", + "type": "plain-text" + } + ], + "bumpFiles": [ + { + "filename": "version.txt", + "type": "plain-text" + } + ] +} diff --git a/images/build-amazonlinux2/.bazelversion b/images/build-amazonlinux2/.bazelversion new file mode 120000 index 0000000..8f79a5a --- /dev/null +++ b/images/build-amazonlinux2/.bazelversion @@ -0,0 +1 @@ +../../etc/.bazelversion \ No newline at end of file diff --git a/images/build-amazonlinux2/Dockerfile b/images/build-amazonlinux2/Dockerfile new file mode 100644 index 0000000..343d001 --- /dev/null +++ b/images/build-amazonlinux2/Dockerfile @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM amazonlinux:2.0.20220912.1 + +COPY /install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/ +COPY get_workspace_mount /usr/local/bin + +ARG TARGETARCH +ENV BUILD_ARCH="${TARGETARCH}" \ + WORKSPACE=/src/workspace + +RUN \ + /scripts/install_apps && \ + /scripts/generate_system_bazelrc --user-root-name amazonlinux2 && \ + /scripts/install_golang_apps && \ + rm -rf /scripts + +ENV PATH="${PATH}:/usr/local/go/bin" diff --git a/images/build-amazonlinux2/generate_system_bazelrc b/images/build-amazonlinux2/generate_system_bazelrc new file mode 120000 index 0000000..8f22303 --- /dev/null +++ b/images/build-amazonlinux2/generate_system_bazelrc @@ -0,0 +1 @@ +../generate_system_bazelrc \ No newline at end of file diff --git a/images/build-amazonlinux2/get_workspace_mount b/images/build-amazonlinux2/get_workspace_mount new file mode 120000 index 0000000..581579a --- /dev/null +++ b/images/build-amazonlinux2/get_workspace_mount @@ -0,0 +1 @@ +../get_workspace_mount \ No newline at end of file diff --git a/images/build-amazonlinux2/install_apps b/images/build-amazonlinux2/install_apps new file mode 100755 index 0000000..80925bc --- /dev/null +++ b/images/build-amazonlinux2/install_apps @@ -0,0 +1,91 @@ +#!/bin/bash + +set -o pipefail +set -o errexit + +VERBOSE=0 + +usage() { + local exitval=${1-1} + cat >&2 << USAGE +usage: + $0 + --verbose Emit verbose info. Default: false +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --verbose) + VERBOSE=1 + shift + ;; + -h | --help) + usage 0 + break + ;; + *) + usage + break + ;; + esac +done + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +readonly SCRIPT_DIR +# shellcheck disable=SC1090 +. "${SCRIPT_DIR}"/install_go.sh + +function yum_update() { + yum -y update +} + +function install_python() { + amazon-linux-extras install -y python3.8 + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 100 +} + +function install_nitro() { + amazon-linux-extras install -y aws-nitro-enclaves-cli + yum install -y aws-nitro-enclaves-cli-devel tar +} + +function install_gcc() { + # install gcc, which is required to build socat + yum install -y gcc-7.3.1-15.amzn2 +} + +function install_misc() { + yum install -y unzip zip +} + +function install_packer() { + yum install -y yum-utils + yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo + yum -y install packer + update-alternatives --install /usr/local/bin/packer packer /usr/bin/packer 100 + + /usr/local/bin/packer version +} + +function install_clang() { + yum install -y clang-11.1.0-1.amzn2.0.2 + + clang --version +} + +if [[ ${VERBOSE} -eq 1 ]]; then + printf "=== SHELL ENVIRONMENT ===\n" + env +fi + +yum_update +install_python +install_nitro +install_misc +install_clang +install_golang "${BUILD_ARCH}" +install_gcc +install_packer diff --git a/images/build-amazonlinux2/install_go.sh b/images/build-amazonlinux2/install_go.sh new file mode 120000 index 0000000..211fd00 --- /dev/null +++ b/images/build-amazonlinux2/install_go.sh @@ -0,0 +1 @@ +../install_go.sh \ No newline at end of file diff --git a/images/build-amazonlinux2/install_golang_apps b/images/build-amazonlinux2/install_golang_apps new file mode 120000 index 0000000..acc9d5a --- /dev/null +++ b/images/build-amazonlinux2/install_golang_apps @@ -0,0 +1 @@ +../install_golang_apps \ No newline at end of file diff --git a/images/build-amazonlinux2/test/commands.yaml b/images/build-amazonlinux2/test/commands.yaml new file mode 100644 index 0000000..41823ae --- /dev/null +++ b/images/build-amazonlinux2/test/commands.yaml @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to docs at https://github.com/GoogleContainerTools/container-structure-test + +schemaVersion: 2.0.0 + +# command tests require the docker toolchain +commandTests: + - name: nitro-cli version + command: "nitro-cli" + args: ["--version"] + exitCode: 0 + + - name: nitro-cli build-enclave help + command: "nitro-cli" + args: ["build-enclave", "--help"] + exitCode: 0 diff --git a/images/build-debian/.bazelversion b/images/build-debian/.bazelversion new file mode 120000 index 0000000..8f79a5a --- /dev/null +++ b/images/build-debian/.bazelversion @@ -0,0 +1 @@ +../../etc/.bazelversion \ No newline at end of file diff --git a/images/build-debian/Dockerfile b/images/build-debian/Dockerfile new file mode 100644 index 0000000..bb3a230 --- /dev/null +++ b/images/build-debian/Dockerfile @@ -0,0 +1,38 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:20.04 + +COPY install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/ +COPY get_workspace_mount /usr/local/bin +COPY gitconfig /etc + +ARG LOCALE=en_US.UTF-8 +ARG TARGETARCH +ENV BUILD_ARCH="${TARGETARCH}" \ + WORKSPACE=/src/workspace \ + TZ=Etc/UTC \ + LANG=${LOCALE} \ + LANGUAGE=${LOCALE} \ + LC_ALL=${LOCALE} \ + LC_CTYPE=${LOCALE} + +RUN \ + chmod 644 /etc/gitconfig && \ + /scripts/install_apps --locale ${LOCALE} && \ + /scripts/generate_system_bazelrc --user-root-name ubuntu && \ + /scripts/install_golang_apps && \ + rm -rf /scripts + +ENV PATH="${PATH}:/usr/local/go/bin" diff --git a/images/build-debian/generate_system_bazelrc b/images/build-debian/generate_system_bazelrc new file mode 120000 index 0000000..8f22303 --- /dev/null +++ b/images/build-debian/generate_system_bazelrc @@ -0,0 +1 @@ +../generate_system_bazelrc \ No newline at end of file diff --git a/images/build-debian/get_workspace_mount b/images/build-debian/get_workspace_mount new file mode 120000 index 0000000..581579a --- /dev/null +++ b/images/build-debian/get_workspace_mount @@ -0,0 +1 @@ +../get_workspace_mount \ No newline at end of file diff --git a/images/build-debian/gitconfig b/images/build-debian/gitconfig new file mode 120000 index 0000000..db62a00 --- /dev/null +++ b/images/build-debian/gitconfig @@ -0,0 +1 @@ +../gitconfig \ No newline at end of file diff --git a/images/build-debian/install_apps b/images/build-debian/install_apps new file mode 100755 index 0000000..6940fea --- /dev/null +++ b/images/build-debian/install_apps @@ -0,0 +1,121 @@ +#!/bin/bash + +set -o pipefail +set -o errexit + +VERBOSE=0 +INSTALL_LOCALE=en_US.UTF-8 + +usage() { + local exitval=${1-1} + cat >&2 << USAGE +usage: + $0 + --locale Set locale. Default: ${INSTALL_LOCALE} + --verbose Emit verbose info. Default: false +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --locale) + INSTALL_LOCALE="$2" + shift + shift + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h | --help) + usage 0 + break + ;; + *) usage 0 ;; + esac +done + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +readonly SCRIPT_DIR +# shellcheck disable=SC1090 +. "${SCRIPT_DIR}"/install_go.sh + +function apt_update() { + apt-get --quiet -o 'Acquire::https::No-Cache=True' -o 'Acquire::http::No-Cache=True' update +} + +function install_python() { + apt-get --quiet install -y python3.8-venv="3.8.10-0ubuntu1~20.04.5" +} + +function install_misc() { + DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + curl \ + git \ + gnupg \ + locales \ + lsb-release \ + software-properties-common \ + wget + if [[ -n ${INSTALL_LOCALE} ]]; then + printf "\nSetting locale to: %s\n" "${INSTALL_LOCALE}" + locale-gen "${INSTALL_LOCALE}" + update-locale LANG="${INSTALL_LOCALE}" + fi +} + +function install_utils() { + DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y --no-install-recommends \ + unzip \ + zip \ + rename +} + +function install_clang() { + declare -r VERSION="14" + curl --silent --fail --show-error --location --remote-name https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + ./llvm.sh ${VERSION} + apt-get --quiet install -y --no-install-recommends libc++-${VERSION}-dev + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${VERSION} 100 + rm -f llvm.sh + + clang --version +} + +# Install Docker (https://docs.docker.com/engine/install/debian/) +function install_docker() { + declare -r arch="$1" + apt-get --quiet remove docker docker.io containerd runc + mkdir -p /etc/apt/keyrings + declare -r DIST=ubuntu + curl --silent --fail --show-error --location https://download.docker.com/linux/${DIST}/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/${DIST} $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list + apt_update + apt-get --quiet install -y --no-install-recommends docker-ce docker-ce-cli containerd.io +} + +function clean_debian() { + apt-get --quiet autoremove -y + apt-get autoclean + apt-get clean + rm -rf /var/lib/apt/lists +} + +if [[ ${VERBOSE} -eq 1 ]]; then + printf "=== SHELL ENVIRONMENT ===\n" + env +fi + +apt_update +install_python +install_misc +install_utils +install_clang +install_golang "${BUILD_ARCH}" +install_docker "${BUILD_ARCH}" +clean_debian diff --git a/images/build-debian/install_go.sh b/images/build-debian/install_go.sh new file mode 120000 index 0000000..211fd00 --- /dev/null +++ b/images/build-debian/install_go.sh @@ -0,0 +1 @@ +../install_go.sh \ No newline at end of file diff --git a/images/build-debian/install_golang_apps b/images/build-debian/install_golang_apps new file mode 120000 index 0000000..acc9d5a --- /dev/null +++ b/images/build-debian/install_golang_apps @@ -0,0 +1 @@ +../install_golang_apps \ No newline at end of file diff --git a/images/build-debian/test/commands.yaml b/images/build-debian/test/commands.yaml new file mode 100644 index 0000000..b3fc2be --- /dev/null +++ b/images/build-debian/test/commands.yaml @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to docs at https://github.com/GoogleContainerTools/container-structure-test + +schemaVersion: 2.0.0 + +# command tests require the docker toolchain +commandTests: + - name: "curl version" + command: "curl" + args: ["--version"] + exitCode: 0 + + - name: "clang version" + command: "clang" + args: ["--version"] + exitCode: 0 + + - name: "bazel version" + command: "bazelisk" + args: ["version"] + exitCode: 0 + + - name: "docker image help" + command: "docker" + args: ["help", "image"] + exitCode: 0 diff --git a/images/generate_system_bazelrc b/images/generate_system_bazelrc new file mode 100755 index 0000000..af82802 --- /dev/null +++ b/images/generate_system_bazelrc @@ -0,0 +1,81 @@ +#!/bin/bash + +# Generate a system bazelrc file +# Designed to be executed when generating a container image, for example from Dockerfile. +# For info on bazelrc files, refer to https://bazel.build/run/bazelrc?hl=en. + +set -o pipefail +set -o errexit + +function usage() { + local exitval=${1-1} + cat &>/dev/stderr << USAGE +usage: + $0 + --user-root-name Name of bazel user-root directory (within bazel cache dir) +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --user-root-name) + USER_ROOT_NAME="$2" + shift + shift + ;; + -h | --help) usage 0 ;; + *) + printf "unrecognized arg: %s\n" "$1" + usage + break + ;; + esac +done + +if [[ -z ${USER_ROOT_NAME} ]]; then + printf -- "error: --user-root-name must be specified\n" &>/dev/stderr + usage 1 +fi + +function _get_toolchains_hash() { + { + # emit versions of all tools relevant to builds + clang --version + /usr/bin/python3 --version + } | sha256sum | cut --delimiter=" " --fields=1 +} + +TOOLCHAINS_HASH=$(_get_toolchains_hash) +readonly TOOLCHAINS_HASH + +readonly BAZELRC="/etc/bazel.bazelrc" +readonly BAZEL_ROOT=/bazel_root +readonly BAZEL_OUTPUT_USER_ROOT="${BAZEL_ROOT}/build_${USER_ROOT_NAME}_${TOOLCHAINS_HASH:0:7}" +mkdir -p "${BAZEL_OUTPUT_USER_ROOT}" + +# Within the running container, globally set these variables. This is less +# obvious than using Dockerfile's ENV, but this particular value isn't +# available to the Dockerfile +# Note: The /etc/profile.d/bazel_env.sh will often be sourced automatically +# but this may only occur for login shells. For bash, the --login +# flag should be supplied. Otherwise, the bazel_env.sh file can be +# sourced directly. +cat >/etc/profile.d/bazel_env.sh <> \$HOME/.bazelrc +EOF + +printf "generating %s\n" "${BAZELRC}" &>/dev/stderr + +# the output_user_root setting will have no effect as long as output_base is set. However, in the +# case that $HOME/.bazelrc is not created via sourcing /etc/profile.d/bazel_env.sh, at least +# the appropriate container-specific output_user_root directory will be used +cat <"${BAZELRC}" +startup --output_user_root="${BAZEL_OUTPUT_USER_ROOT}" +# set a variable based on the hash of all build tool dependencies other than bazel itself +# primarily to avoid spurious cache hits for distinct sets of toolchains +build --action_env=TOOLCHAINS_HASH=${TOOLCHAINS_HASH} +BAZELRC diff --git a/images/get_workspace_mount b/images/get_workspace_mount new file mode 100755 index 0000000..517b2ce --- /dev/null +++ b/images/get_workspace_mount @@ -0,0 +1,35 @@ +#!/bin/bash + +function get_docker_workspace_mount() { + # when running inside a docker container, expect /.dockerenv to exist + if ! [[ -f /.dockerenv ]]; then + printf "Error: %s must execute inside a docker container\n" "$(basename "${BASH_SOURCE[0]}")" &>/dev/stderr + exit 1 + fi + + # if running inside a docker container, the workspace mount point cannot be + # determined by git or bazel or inspecting the filesystem itself. Instead, we + # need to use docker to expose its mount info for the /src/workspace path. + # determine the current container's ID + local -r CONTAINER_ID="$(uname --nodename)" + # use docker inspect to extract the current mount path for /src/workspace + # this format string is a golang template (https://pkg.go.dev/text/template) processed + # by docker's --format flag, per https://docs.docker.com/config/formatting/ + # shellcheck disable=SC2016 + local -r FORMAT_STR=' + {{- range $v := .HostConfig.Binds -}} + {{$pathpair := split $v ":" -}} + {{if eq (index $pathpair 1) "/src/workspace" -}} + {{print (index $pathpair 0) -}} + {{end -}} + {{end -}} + ' + local -r MOUNT_PATH="$(docker inspect --format "${FORMAT_STR}" "${CONTAINER_ID}")" + if [[ -z ${MOUNT_PATH} ]]; then + printf "Error: Unable to determine mount point for /src/workspace. Exiting\n" &>/dev/stderr + exit 1 + fi + printf "%s" "${MOUNT_PATH}" +} + +get_docker_workspace_mount diff --git a/images/gitconfig b/images/gitconfig new file mode 100644 index 0000000..eaf257f --- /dev/null +++ b/images/gitconfig @@ -0,0 +1,2 @@ +[safe] + directory = /src/workspace diff --git a/images/install_go.sh b/images/install_go.sh new file mode 100644 index 0000000..c284b13 --- /dev/null +++ b/images/install_go.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shell library to install and uninstall golang + +function _golang_install_dir() { + printf "/usr/local/go\n" +} + +function install_golang() { + declare -r _ARCH="$1" + declare -r FNAME=gobin.tar.gz + declare -r VERSION=1.19 + # shellcheck disable=SC2155 + declare -r GO_INSTALL_DIR="$(_golang_install_dir)" + declare -r -A GO_HASHES=( + [amd64]="464b6b66591f6cf055bc5df90a9750bf5fbc9d038722bb84a9d56a2bea974be6" + [arm64]="efa97fac9574fc6ef6c9ff3e3758fb85f1439b046573bf434cccb5e012bd00c8" + ) + declare -r GO_HASH=${GO_HASHES[${_ARCH}]} + if [[ -z ${GO_HASH} ]]; then + printf "Unrecognized or unsupported architecture for golang: %s\n" "${_ARCH}" &>/dev/stderr + exit 1 + fi + + curl --silent -fSL --output ${FNAME} "https://go.dev/dl/go${VERSION}.linux-${_ARCH}.tar.gz" + echo "${GO_HASH} ${FNAME}" | sha256sum -c + tar --directory /usr/local -xzf ${FNAME} + rm -f ${FNAME} + update-alternatives --install /usr/bin/go go "${GO_INSTALL_DIR}"/bin/go 100 + + go version +} + +function remove_golang() { + update-alternatives --remove-all go + rm -rf "$(_golang_install_dir)" +} diff --git a/images/install_golang_apps b/images/install_golang_apps new file mode 100755 index 0000000..a10d33e --- /dev/null +++ b/images/install_golang_apps @@ -0,0 +1,46 @@ +#!/bin/bash + +set -o pipefail +set -o errexit + +BAZEL_PATH=/usr/bin + +usage() { + local exitval=${1-1} + cat >&2 << USAGE +usage: + $0 + --verbose Emit verbose info. Default: false +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --verbose) + set -o xtrace + shift + ;; + -h | --help) usage 0 ;; + *) + usage + break + ;; + esac +done + +function install_bazelisk() { + go install github.com/bazelbuild/bazelisk@v1.13.2 + BAZELISK="$(go env GOPATH)"/bin/bazelisk + if [[ -n ${BAZEL_PATH} ]] && [[ -d ${BAZEL_PATH} ]]; then + ln -s "${BAZELISK}" "${BAZEL_PATH}"/bazel + fi + # install the specified version of bazel + USE_BAZEL_VERSION="$(cat /scripts/.bazelversion)" + export USE_BAZEL_VERSION + "${BAZELISK}" version + rm -rf /bazel_root/* +} + +install_bazelisk diff --git a/images/presubmit/.pre-commit-config.yaml b/images/presubmit/.pre-commit-config.yaml new file mode 120000 index 0000000..9971a06 --- /dev/null +++ b/images/presubmit/.pre-commit-config.yaml @@ -0,0 +1 @@ +../../etc/.pre-commit-config.yaml \ No newline at end of file diff --git a/images/presubmit/Dockerfile b/images/presubmit/Dockerfile new file mode 100644 index 0000000..b5c96c6 --- /dev/null +++ b/images/presubmit/Dockerfile @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:20.04 + +COPY install_apps install_go.sh .pre-commit-config.yaml /scripts/ +COPY gitconfig /etc + +ARG PRE_COMMIT_VENV_DIR=/usr/pre-commit-venv +ARG TARGETARCH +ENV BUILD_ARCH="${TARGETARCH}" \ + WORKSPACE=/src/workspace \ + PRE_COMMIT_HOME=/var/cache/pre-commit \ + PRE_COMMIT_TOOL=${PRE_COMMIT_VENV_DIR}/bin/pre-commit \ + TZ=Etc/UTC + +RUN \ + chmod 644 /etc/gitconfig && \ + /usr/bin/env -v PRE_COMMIT_VENV_DIR=${PRE_COMMIT_VENV_DIR} /scripts/install_apps --verbose && \ + rm -rf /scripts + +ENV PATH="${PATH}:/usr/local/go/bin" diff --git a/images/presubmit/gitconfig b/images/presubmit/gitconfig new file mode 120000 index 0000000..db62a00 --- /dev/null +++ b/images/presubmit/gitconfig @@ -0,0 +1 @@ +../gitconfig \ No newline at end of file diff --git a/images/presubmit/install_apps b/images/presubmit/install_apps new file mode 100755 index 0000000..f9bc2d8 --- /dev/null +++ b/images/presubmit/install_apps @@ -0,0 +1,122 @@ +#!/bin/bash + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail +set -o errexit + +VERBOSE=0 + +usage() { + local exitval=${1-1} + cat >&2 << USAGE +usage: + $0 + --verbose Emit verbose info. Default: false +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --verbose) + VERBOSE=1 + set -o xtrace + shift + ;; + -h | --help) + usage 0 + break + ;; + *) usage 0 ;; + esac +done + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +readonly SCRIPT_DIR + +function apt_update() { + apt-get --quiet -o 'Acquire::https::No-Cache=True' -o 'Acquire::http::No-Cache=True' update +} + +function install_python() { + apt-get --quiet install -y python3.8-venv="3.8.10-0ubuntu1~20.04.5" +} + +function install_packages() { + declare -a PACKAGES=( + lsb-release + apt-transport-https + ca-certificates + wget + curl + gnupg + software-properties-common + shellcheck + ) + DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y "${PACKAGES[@]}" +} + +# Install Docker (https://docs.docker.com/engine/install/debian/) +function install_docker() { + apt-get --quiet remove docker docker.io containerd runc + mkdir -p /etc/apt/keyrings + DIST=ubuntu + curl --silent --fail --show-error --location https://download.docker.com/linux/${DIST}/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo "deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/${DIST} $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list + apt_update + DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y docker-ce docker-ce-cli containerd.io +} + +function install_precommit_cache() { + python3 -m venv "${PRE_COMMIT_VENV_DIR}" + "${PRE_COMMIT_VENV_DIR}"/bin/pip install pre-commit + "${PRE_COMMIT_TOOL}" --version + + # initialize pre-commit cache, which needs a git repo (a temporary will suffice) + GIT_REPO="$(mktemp -d)" + readonly GIT_REPO + git init "${GIT_REPO}" + cd "${GIT_REPO}" + # create a placeholder terraform file in the git repo, giving addlicense something to do + echo "# a placeholder file for addlicense" > placeholder.tf + git add placeholder.tf + # run a single arbitrary hook to trigger population of cache + "${PRE_COMMIT_TOOL}" run --config "${SCRIPT_DIR}"/.pre-commit-config.yaml || true + rm -rf "${GIT_REPO}" +} + +if [[ ${VERBOSE} -eq 1 ]]; then + printf "=== SHELL ENVIRONMENT ===\n" + env +fi + +ARCH=$(dpkg --print-architecture) +readonly ARCH + +apt_update +install_python +install_packages + +# shellcheck disable=SC1090 +. "${SCRIPT_DIR}"/install_go.sh +install_golang "${ARCH}" +install_docker +install_precommit_cache +remove_golang + +apt-get --quiet autoremove -y +apt-get clean diff --git a/images/presubmit/install_go.sh b/images/presubmit/install_go.sh new file mode 120000 index 0000000..211fd00 --- /dev/null +++ b/images/presubmit/install_go.sh @@ -0,0 +1 @@ +../install_go.sh \ No newline at end of file diff --git a/images/release/Dockerfile b/images/release/Dockerfile new file mode 100644 index 0000000..3e60018 --- /dev/null +++ b/images/release/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM node:18.9.0-alpine3.16 + +COPY install_release_apps /scripts/ +COPY gitconfig /etc + +ARG TARGETARCH +ENV BUILD_ARCH="${TARGETARCH}" \ + WORKSPACE=/src/workspace + +RUN \ + chmod 644 /etc/gitconfig && \ + /scripts/install_release_apps && \ + rm -rf /scripts + +ENTRYPOINT ["/bin/sh"] diff --git a/images/release/gitconfig b/images/release/gitconfig new file mode 100644 index 0000000..a72cd19 --- /dev/null +++ b/images/release/gitconfig @@ -0,0 +1,3 @@ +[user] + email = nobody@google.com + name = Privacy Sandbox Release System diff --git a/images/release/install_release_apps b/images/release/install_release_apps new file mode 100755 index 0000000..204c8c5 --- /dev/null +++ b/images/release/install_release_apps @@ -0,0 +1,6 @@ +#!/bin/sh + +npm install --global commit-and-tag-version + +# Install the GitHub CLI tool (https://cli.github.com/) +apk add github-cli diff --git a/images/release/test/commands.yaml b/images/release/test/commands.yaml new file mode 100644 index 0000000..62ac663 --- /dev/null +++ b/images/release/test/commands.yaml @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to docs at https://github.com/GoogleContainerTools/container-structure-test + +schemaVersion: 2.0.0 + +# command tests require the docker toolchain +commandTests: + - name: npm version + command: "npm" + args: ["--version"] + exitCode: 0 + + - name: commit-and-tag-version help + command: "commit-and-tag-version" + args: ["--help"] + exitCode: 0 diff --git a/images/release/test/structure.yaml b/images/release/test/structure.yaml new file mode 100644 index 0000000..1d70464 --- /dev/null +++ b/images/release/test/structure.yaml @@ -0,0 +1,23 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to docs at https://github.com/GoogleContainerTools/container-structure-test + +schemaVersion: 2.0.0 + +fileExistenceTests: + - name: git config + path: /etc/gitconfig + shouldExist: true + permissions: "-r--r--r--" diff --git a/images/utils/Dockerfile b/images/utils/Dockerfile new file mode 100644 index 0000000..0768f6c --- /dev/null +++ b/images/utils/Dockerfile @@ -0,0 +1,19 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.16 + +RUN apk --no-cache add \ + unzip=6.0-r9 \ + zip=3.0-r9 diff --git a/tools/aws-cli b/tools/aws-cli new file mode 100755 index 0000000..44beb2e --- /dev/null +++ b/tools/aws-cli @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# environment variables (all optional): +# WORKSPACE repo root directory, must be an absolute path +# +# environment variables exported to AWS CLI +# For more info on supported env vars, see: +# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html +# +# AWS_ACCESS_KEY_ID +# AWS_SECRET_ACCESS_KEY +# AWS_SESSION_TOKEN +# AWS_REGION +# AWS_DEFAULT_REGION +# AWS_PROFILE + +set -o errexit + +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"/builder.sh + +declare -a ENV_VARS +builder::add_aws_env_vars ENV_VARS +ENV_VARS+=( + "HOME=/home" +) + +declare -a DOCKER_RUN_ARGS +DOCKER_RUN_ARGS+=( + "--rm" +) +for evar in "${ENV_VARS[@]}" +do + DOCKER_RUN_ARGS+=( + "--env ${evar}" + ) +done +if [[ -t 0 ]] && [[ -t 1 ]]; then + # stdin and stdout are open, assume it's an interactive tty session + DOCKER_RUN_ARGS+=( + "--interactive" + "--tty" + ) +fi + +REL_PWD="$(realpath --relative-to="${WORKSPACE}" "$(pwd)")" +readonly REL_PWD + +# shellcheck disable=SC2068 +docker run \ + ${DOCKER_RUN_ARGS[@]} \ + --user "$(id -u):$(id -g)" \ + --volume "${HOME}"/.aws/:/home/.aws/ \ + --volume "${WORKSPACE}":/src/workspace \ + --workdir /src/workspace/"${REL_PWD}" \ + amazon/aws-cli -- "$@" diff --git a/tools/bazel-amazonlinux2 b/tools/bazel-amazonlinux2 new file mode 120000 index 0000000..98d6d30 --- /dev/null +++ b/tools/bazel-amazonlinux2 @@ -0,0 +1 @@ +bazel-debian \ No newline at end of file diff --git a/tools/bazel-debian b/tools/bazel-debian new file mode 100755 index 0000000..5cb4313 --- /dev/null +++ b/tools/bazel-debian @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# environment variables supported by cbuild (all optional): +# WORKSPACE Set the path to the workspace (repo root) +# AWS_ACCESS_KEY_ID AWS auth token +# AWS_SECRET_ACCESS_KEY AWS auth token +# BAZEL_STARTUP_ARGS Additional startup arguments to pass to bazel invocations +# BAZEL_EXTRA_ARGS Additional command arguments to pass to bazel invocations +# EXTRA_DOCKER_RUN_ARGS Additional arguments to pass to docker run invocations + +set -o pipefail +set -o errexit + +SCRIPT_NAME=$(basename "$0") +CBUILD="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/cbuild" +IMAGE="build-${SCRIPT_NAME/bazel-}" + +# shellcheck disable=SC1083 +"${CBUILD}" --image "${IMAGE}" --cmd " +printf \"bazel output_base: [%s]\n\" \"\$(bazel info output_base)\" +bazel ${BAZEL_STARTUP_ARGS} $* ${BAZEL_EXTRA_ARGS} +" diff --git a/tools/builder.sh b/tools/builder.sh new file mode 100644 index 0000000..ce2decb --- /dev/null +++ b/tools/builder.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# shell package enabling support for docker +# + +# require this script to be sourced rather than executed +if ! (return 0 2>/dev/null); then + printf "Error: Script %s must be sourced\n" "${BASH_SOURCE[0]}" &>/dev/stderr + exit 1 +fi + +####################################### +# Configure and export the WORKSPACE variable. Does not +# overwrite existing WORKSPACE value. +####################################### +function builder::set_workspace() { + local GIT_TOPLEVEL="$(git rev-parse --show-superproject-working-tree)" + if [[ -n ${GIT_TOPLEVEL} ]]; then + WORKSPACE="${GIT_TOPLEVEL}" + elif [[ -z ${WORKSPACE} ]]; then + WORKSPACE="$(git rev-parse --show-toplevel)" + fi + export WORKSPACE +} + +####################################### +# Return the path to the build workspace, for use with +# the docker volume or mount argument +####################################### +function builder::get_docker_workspace_mount() { + # when running inside a docker container, expect /.dockerenv to exist + if ! [[ -f /.dockerenv ]]; then + printf "%s" "${WORKSPACE}" + return + fi + + # if running inside a docker container, the workspace mount point cannot be + # determined by git or bazel or inspecting the filesystem itself. Instead, we + # need to use docker to expose its mount info for the /src/workspace path. + # determine the current container's ID + declare CONTAINER_ID + CONTAINER_ID="$(uname --nodename)" + # use docker inspect to extract the current mount path for /src/workspace + # this format string is a golang template (https://pkg.go.dev/text/template) processed + # by docker's --format flag, per https://docs.docker.com/config/formatting/ + # shellcheck disable=SC2016 + declare -r FORMAT_STR=' + {{- range $v := .HostConfig.Binds -}} + {{$pathpair := split $v ":" -}} + {{if eq (index $pathpair 1) "/src/workspace" -}} + {{print (index $pathpair 0) -}} + {{end -}} + {{end -}} + ' + # shellcheck disable=SC2155 + declare -r MOUNT_PATH="$(docker inspect --format "${FORMAT_STR}" "${CONTAINER_ID}")" + if [[ -z ${MOUNT_PATH} ]]; then + printf "Error: Unable to determine mount point for /src/workspace. Exiting\n" &>/dev/stderr + exit 1 + fi + printf "%s" "${MOUNT_PATH}" +} + +function builder::get_tools_dir() { + dirname "$(readlink -f "${BASH_SOURCE[0]}")" +} + +####################################### +# Invoke cbuild tool in a build-debian container +####################################### +function builder::cbuild_debian() { + local -r CBUILD="$(builder::get_tools_dir)"/cbuild + printf "=== cbuild debian action envs ===\n" + # shellcheck disable=SC2086 + "${CBUILD}" ${CBUILD_ARGS} --image build-debian --cmd "grep -o 'action_env.*' /etc/bazel.bazelrc 1>/dev/stderr 2>/dev/null" + # shellcheck disable=SC2086 + "${CBUILD}" ${CBUILD_ARGS} --image build-debian --cmd "$*" +} + +####################################### +# Add AWS env vars to a specified array +# Arguments: +# * the name of an array to which to append values +####################################### +function builder::add_aws_env_vars() { + declare -n args=$1 + args+=( + "AWS_ACCESS_KEY_ID" + "AWS_SECRET_ACCESS_KEY" + "AWS_SESSION_TOKEN" + "AWS_REGION" + "AWS_DEFAULT_REGION" + "AWS_PROFILE" + ) +} + +####################################### +# Invoke cbuild tool in a build-amazonlinux2 container +####################################### +function builder::cbuild_al2() { + local -r CBUILD="$(builder::get_tools_dir)"/cbuild + declare -a env_vars + builder::add_aws_env_vars env_vars + declare env_args + for evar in "${env_vars[@]}" + do + env_args+=("--env" "${evar}") + done + set -o xtrace + printf "=== cbuild amazonlinux2 action envs ===\n" + # shellcheck disable=SC2086 + "${CBUILD}" ${CBUILD_ARGS} "${env_args[@]}" --image build-amazonlinux2 --cmd "grep -o 'action_env.*' /etc/bazel.bazelrc 1>/dev/stderr 2>/dev/null" + # shellcheck disable=SC2086 + "${CBUILD}" ${CBUILD_ARGS} "${env_args[@]}" --image build-amazonlinux2 --cmd "$*" +} + +####################################### +# Return the numeric id of the user or group +# Arguments: +# single character, either "u" for user or "g" for group +####################################### +function builder::id() { + declare -r mode="$1" + declare -i _id + _id=$(id "-${mode}") + if [[ ${_id} -ne 0 ]]; then + printf "%s" "${_id}" + else + # id is 0 (root), use the owner/group of the WORKSPACE file instead + stat -c "%${mode}" "${WORKSPACE}"/WORKSPACE + fi +} + +####################################### +# invoke functions to configure the build environment +####################################### + +builder::set_workspace diff --git a/tools/cbuild b/tools/cbuild new file mode 100755 index 0000000..e6ef366 --- /dev/null +++ b/tools/cbuild @@ -0,0 +1,187 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail +set -o errexit + +IMAGE="build-debian" +VERBOSE=0 +declare -r -a IMAGE_LIST=( + "build-debian" + "build-amazonlinux2" + "presubmit" +) + +declare -a ENV_VARS +ENV_VARS+=( + "BAZEL_STARTUP_ARGS" + "BAZEL_EXTRA_ARGS" +) + +usage() { + local exitval=${1-1} + cat &>/dev/stderr << USAGE +usage: + $0 + --cmd Command(s) to execute within the docker container, via \`bash -c\` + --image Image name for the build runtime. Valid names: +USAGE + + for elem in "${IMAGE_LIST[@]}" + do + printf " * %s\n" "${elem}" &>/dev/stderr + done + + cat &>/dev/stderr << USAGE + --env [=] Name (or name=value) of exported environment variable, propagated into container + --without-shared-cache Containers will not mount ${HOME}/.cache/bazel + --without-embedded-docker Disable docker client within container + --verbose Enable verbose output + +Environment variables (all optional): + WORKSPACE Full path to the workspace (repo root) + EXTRA_DOCKER_RUN_ARGS Additional arguments to pass to docker run invocations + +Environment variables propagated into container: + BAZEL_STARTUP_ARGS + BAZEL_EXTRA_ARGS +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +WITH_SHARED_CACHE=1 +WITH_DOCKER_SOCK=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + --cmd) + CMD="$2" + shift + shift + ;; + --env) + ENV_VARS+=("$2") + shift + shift + ;; + --image) + IMAGE="$2" + shift + shift + ;; + --without-shared-cache) + WITH_SHARED_CACHE=0 + shift + ;; + --without-embedded-docker) + WITH_DOCKER_SOCK=0 + shift + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h | --help) + usage 0 + break + ;; + *) + printf "unrecognized arg: %s\n" "$1" + usage + break + ;; + esac +done + +if [[ -z ${IMAGE} ]]; then + printf -- "error: --image must be specified\n" &>/dev/stderr + usage 1 +fi +# shellcheck disable=SC2076 +if ! [[ " ${IMAGE_LIST[*]} " =~ " ${IMAGE} " ]]; then + printf -- "error: image [%s] not recognized\n" "${IMAGE}" &>/dev/stderr + usage 1 +fi + +TOOLS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +readonly TOOLS_DIR +# shellcheck disable=SC1091 +source "${TOOLS_DIR}"/builder.sh + +WORKSPACE_MOUNT="$(builder::get_docker_workspace_mount)" +readonly WORKSPACE_MOUNT +if [[ ${VERBOSE} -eq 1 ]]; then + printf "mounting workspace into container: %s\n" "${WORKSPACE_MOUNT}" &>/dev/stderr +fi + +IMAGE_TAGGED="$("${TOOLS_DIR}"/get-builder-image-tagged --image "${IMAGE}")" +readonly IMAGE_TAGGED + +declare -a DOCKER_RUN_ARGS +DOCKER_RUN_ARGS+=( + "--rm" + "--entrypoint=/bin/bash" + "--volume ${WORKSPACE_MOUNT}:/src/workspace" + "--workdir /src/workspace" + "${EXTRA_DOCKER_RUN_ARGS}" +) + +# inside the docker build images, /bazel_root is the bazel cache dir, per the system-wide bazelrc +readonly BAZEL_ROOT=/bazel_root +if [[ ${WITH_SHARED_CACHE} -eq 0 ]]; then + # use tmpfs for as temporary, container-bound bazel cache + DOCKER_RUN_ARGS+=( + "--tmpfs ${BAZEL_ROOT}:exec" + ) +else + # mount host filesystem for "shared" use by multiple docker container invocations + DOCKER_RUN_ARGS+=( + "--volume ${HOME}/.cache/bazel:${BAZEL_ROOT}" + ) +fi +if [[ ${WITH_DOCKER_SOCK} -eq 1 ]]; then + DOCKER_RUN_ARGS+=( + "--volume /var/run/docker.sock:/var/run/docker.sock" + ) +fi +for evar in "${ENV_VARS[@]}" +do + DOCKER_RUN_ARGS+=( + "--env ${evar}" + ) +done +if [[ -t 0 ]] && [[ -t 1 ]]; then + # stdin and stdout are open, assume it's an interactive tty session + DOCKER_RUN_ARGS+=( + "--interactive" + "--tty" + ) +fi + +if [[ ${VERBOSE} -eq 1 ]]; then + set -o xtrace +fi +if [[ -z ${CMD} ]]; then + # shellcheck disable=SC2068 + docker run \ + ${DOCKER_RUN_ARGS[@]} \ + "${IMAGE_TAGGED}" +else + # shellcheck disable=SC2068 + docker run \ + ${DOCKER_RUN_ARGS[@]} \ + "${IMAGE_TAGGED}" \ + --login -c "$CMD" +fi diff --git a/tools/get-architecture b/tools/get-architecture new file mode 100755 index 0000000..a441560 --- /dev/null +++ b/tools/get-architecture @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Print the CPU architecture + +set -o errexit + +trap _cleanup EXIT +function _cleanup() { + local -r -i STATUS=$? + if [[ ${STATUS} -ne 0 ]]; then + printf "Error: get-architecture status code: %s\n" "${STATUS}" &>/dev/stderr + fi + exit ${STATUS} +} + +if [[ -n ${BUILD_ARCH} ]]; then + printf "%s\n" "${BUILD_ARCH}" +else + docker run --rm --entrypoint=/usr/bin/dpkg ubuntu:20.04 --print-architecture +fi diff --git a/tools/get-builder-image-tagged b/tools/get-builder-image-tagged new file mode 100755 index 0000000..ea76164 --- /dev/null +++ b/tools/get-builder-image-tagged @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Return an image:tag for the specified builder docker image +# Given the input files (Dockerfile, configs, installation scripts etc) for building the docker +# image, first generate a hash of the file contents (ie. exclude timestamps, file ownership etc). +# This hash will be used as the image tag + +set -o pipefail +set -o errexit + +VERBOSE=0 +BUILD_IMAGE_IF_NEEDED=1 + +declare -r -A IMAGE_NAMES=( + [build-debian]="privacysandbox/builders/build-debian" + [build-amazonlinux2]="privacysandbox/builders/build-amazonlinux2" + [presubmit]="privacysandbox/builders/presubmit" + [release]="privacysandbox/builders/release" + [utils]="privacysandbox/builders/utils" +) + +usage() { + local exitval=${1-1} + cat &>/dev/stderr << USAGE +usage: + $0 + --no-build Do not build image if it doesn't exist + --image Image name for the build runtime. Valid names: +USAGE + + for elem in "${!IMAGE_NAMES[@]}" + do + if [[ ${IMAGE} == "${elem}" ]]; then + local EXTRA=" (default)" + fi + printf " * %s%s\n" "${elem}" "${EXTRA}" &>/dev/stderr + done + + cat &>/dev/stderr << USAGE + --verbose Enable verbose output +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --image) + IMG="$2" + shift + shift + ;; + --no-build) + BUILD_IMAGE_IF_NEEDED=0 + shift + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h | --help) usage 0 ;; + *) + printf "unrecognized arg: %s\n" "$1" + usage + ;; + esac +done + +TOOLS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +readonly TOOLS_DIR +readonly IMAGES_DIR="${TOOLS_DIR}"/../images +# shellcheck disable=SC1091 +source "${TOOLS_DIR}"/builder.sh + +if [[ -z ${IMG} ]]; then + printf -- "error: --image must be specified\n" &>/dev/stderr + usage 1 +fi + +readonly IMAGE_NAME=${IMAGE_NAMES[${IMG}]} +if [[ -z ${IMAGE_NAME} ]]; then + printf -- "error: image [%s] not recognized\n" "${IMG}" &>/dev/stderr + usage 1 +fi + +IMAGE_PATH_FULL="${IMAGES_DIR}/${IMG}" +if ! [[ -s ${IMAGE_PATH_FULL}/Dockerfile ]]; then + printf "error: unable to locate [%s/Dockerfile]\n" "${IMAGE_PATH_FULL}" &>/dev/stderr + exit 1 +fi +TEMPTAR="$(mktemp --dry-run --suffix=.tar.gz)" +readonly TEMPTAR + +trap _cleanup EXIT + +function _cleanup() { + local STATUS=$? + rm -f "${TEMPTAR}" + # shellcheck disable=SC2086 + exit ${STATUS} +} + +# try to create a deterministic tar file +tar --create --dereference --sort=name --owner=0 --group=0 --numeric-owner --mtime='2022-07-01 12:00Z' --format=gnu --directory="${IMAGE_PATH_FULL}" --file="${TEMPTAR}" --exclude=test . +# use the tarfile size and file content to generate a sha256 hash +SHA=$({ echo -ne "blob $(wc -c < "${TEMPTAR}")\0"; tar --extract --file="${TEMPTAR}" --to-stdout; } | sha256sum | cut -f1 -d" ") +ARCH="$("${TOOLS_DIR}"/get-architecture)" +readonly ARCH +readonly IMAGE_TAG="${ARCH}-${SHA}" +if [[ ${VERBOSE} -eq 1 ]]; then + printf "tag for docker inputs: %s\n" "${IMAGE_TAG}" &>/dev/stderr +fi +readonly IMAGE_TAGGED="${IMAGE_NAME}:${IMAGE_TAG}" +printf "%s\n" "${IMAGE_TAGGED}" + +if [[ ${BUILD_IMAGE_IF_NEEDED} -eq 1 ]]; then + # Create a builder docker image + # build container image and load it into the local docker client + if ! docker image inspect "${IMAGE_TAGGED}" &>/dev/null; then + printf "generating docker image %s\n" "${IMAGE_TAGGED}" &>/dev/stderr + docker buildx build --tag "${IMAGE_TAGGED}" - <"${TEMPTAR}" &>/dev/stderr + fi +fi diff --git a/tools/git-hooks/pre-commit b/tools/git-hooks/pre-commit new file mode 100755 index 0000000..21d5671 --- /dev/null +++ b/tools/git-hooks/pre-commit @@ -0,0 +1,18 @@ +#!/bin/bash + +TOP_LEVEL_DIR="$(git rev-parse --show-toplevel)" +GIT_HOOKS_DIR="${TOP_LEVEL_DIR}/$(git rev-parse --git-dir)/hooks" +declare -a ARGS=( + hook-impl + "--hook-type=pre-commit" + --hook-dir "${GIT_HOOKS_DIR}" + -- + "$@" +) + +if command -v "${TOP_LEVEL_DIR}"/builders/tools/pre-commit > /dev/null; then + exec "${TOP_LEVEL_DIR}"/builders/tools/pre-commit "${ARGS[@]}" +else + printf "[builders/tools/pre-commit] not found" 2>&1 + exit 1 +fi diff --git a/tools/hadolint b/tools/hadolint new file mode 100755 index 0000000..c039521 --- /dev/null +++ b/tools/hadolint @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit + +TOOLS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +readonly TOOLS_DIR +# shellcheck disable=SC1091 +source "${TOOLS_DIR}"/builder.sh + +declare -a DOCKER_RUN_ARGS +DOCKER_RUN_ARGS+=( + "--rm" +) +if [[ -t 0 ]] && [[ -t 1 ]]; then + # stdin and stdout are open, assume it's an interactive tty session + DOCKER_RUN_ARGS+=( + "--interactive" + "--tty" + ) +fi + +WORKSPACE_MOUNT="$(builder::get_docker_workspace_mount)" +readonly WORKSPACE_MOUNT + +docker run \ + "${DOCKER_RUN_ARGS[@]}" \ + --user "$(id -u):$(id -g)" \ + --volume "${WORKSPACE_MOUNT}":/src \ + --workdir /src \ + ghcr.io/hadolint/hadolint:v2.10.0 \ + hadolint "$@" diff --git a/tools/normalize-dist b/tools/normalize-dist new file mode 100755 index 0000000..b05cdd2 --- /dev/null +++ b/tools/normalize-dist @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# environment variables supported by cbuild (all optional): +# WORKSPACE Set the path to the workspace (repo root) +# EXTRA_DOCKER_RUN_ARGS Additional arguments to pass to docker run invocations + +set -o pipefail +set -o errexit + +trap _cleanup EXIT +function _cleanup() { + local -r -i STATUS=$? + if [[ ${STATUS} -eq 0 ]]; then + printf "normalize-dist completed successfully\n" &>/dev/stderr + else + printf "Error: normalize-dist completed with status code: %s\n" "${STATUS}" &>/dev/stderr + fi + exit 0 +} + +TOOLS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +readonly TOOLS_DIR +# shellcheck disable=SC1091 +source "${TOOLS_DIR}"/builder.sh +readonly IMAGE=build-debian +GROUP="$(builder::id g)" +readonly GROUP +USER="$(builder::id u)" +readonly USER + +readonly TOP_LEVEL_DIRS="dist" + +printf "Setting file ownership [%s], group [%s] in dirs [%s]\n" "${USER}" "${GROUP}" "${TOP_LEVEL_DIRS}" +"${TOOLS_DIR}"/cbuild --image ${IMAGE} --cmd " +for TOP_LEVEL_DIR in ${TOP_LEVEL_DIRS}; do + find \${TOP_LEVEL_DIR} -type f -exec chmod 644 {} \; + find \${TOP_LEVEL_DIR} -type d -exec chmod 755 {} \; + chgrp --recursive ${GROUP} \${TOP_LEVEL_DIR} + chown --recursive ${USER} \${TOP_LEVEL_DIR} +done +" diff --git a/tools/pre-commit b/tools/pre-commit new file mode 100755 index 0000000..15144ca --- /dev/null +++ b/tools/pre-commit @@ -0,0 +1,154 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail +set -o errexit + +TOOLS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +readonly TOOLS_DIR +# shellcheck disable=SC1091 +source "${TOOLS_DIR}"/builder.sh + +trap __cleanup EXIT +function __cleanup() { + declare -r -i STATUS=$? + readonly LOGFILE=/var/cache/pre-commit/pre-commit.log + if [[ -s ${LOGFILE} ]]; then + printf "==== %s ====\n" ${LOGFILE} &>/dev/stderr + cat ${LOGFILE} &>/dev/stderr + printf "===========\n" &>/dev/stderr + fi + if [[ ${CLEANUP} -eq 0 ]]; then + exit ${STATUS} + fi + # shellcheck disable=SC2086 + "${TOOLS_DIR}"/cbuild ${CBUILD_COMMON_ARGS} --cmd $" +# change file ownership back to user +{ + git ls-files . --modified + git diff --staged --name-only +} | \ + sort -u | \ + xargs --replace={} /bin/bash -c '{ chown -f $(builder::id u) {}; chgrp -f $(builder::id g) {};}' +# clean up 'empty' node_modules dir created by prettier 2.x +if [[ -d node_modules ]]; then + rmdir node_modules/.cache/prettier/ node_modules/.cache/ node_modules/ +fi +" + exit ${STATUS} +} + +function __exit_msg() { + declare -r MSG="$1" + printf "%s. Exiting\n" "${MSG}" &>/dev/stderr + exit 1 +} + +function __install_git_hooks() { + git rev-parse --is-inside-work-tree >/dev/null || __exit_msg "Not in a git repository" + GIT_HOOKS_SRC_DIR="${TOOLS_DIR}/git-hooks" + GIT_HOOKS_DIR="${WORKSPACE}/.git/hooks" + chmod +x "${GIT_HOOKS_SRC_DIR}" + cp -p "${GIT_HOOKS_SRC_DIR}"/* "${GIT_HOOKS_DIR}" + printf "installed git hooks\n" +} + +function __usage() { + declare -r -i exitval=${1-1} + cat &>/dev/stderr << USAGE +usage: + $0 [command] + +A CLI tool to run pre-commit checks. + + if command not specified, run the pre-commit tool on all files in entire workspace + install install as a git pre-commit hook in the current git repo + autoupdate update hook versions in .pre-commit-config.yaml + +environment variables (all optional): + SKIP comma-delimited list of hook ids to skip +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +function __init() { + IMAGE_TAGGED="$("${TOOLS_DIR}"/get-builder-image-tagged --image presubmit --verbose)" + readonly IMAGE_TAGGED + + local -r ARCH="$("${TOOLS_DIR}"/get-architecture)" + if [[ ${ARCH} == arm64 ]]; then + if [[ -z ${SKIP} ]]; then + SKIP_HOOKS="terraform-fmt" + else + SKIP_HOOKS="${SKIP},terraform-fmt" + fi + else + SKIP_HOOKS="${SKIP}" + fi + if [[ -n ${SKIP_HOOKS} ]]; then + printf "Skipping pre-commit hooks: %s\n" "${SKIP_HOOKS}" + fi + SKIP_ENV="SKIP=${SKIP_HOOKS}" +} + +CLEANUP=0 + +CBUILD_COMMON_ARGS="--without-shared-cache --image presubmit" + +# TODO: run bazel //:precommit-hooks rather than just the pre-commit tool +if [[ $# -gt 0 ]]; then + case "$1" in + install) __install_git_hooks ;; + + help | --help | -h) __usage 0 ;; + + autoupdate) + PRECOMMIT_CMD="$1" + shift + __init + # shellcheck disable=SC2086 + "${TOOLS_DIR}"/cbuild ${CBUILD_COMMON_ARGS} --env "${SKIP_ENV}" --cmd "/usr/pre-commit-venv/bin/pre-commit ${PRECOMMIT_CMD} --config ./.pre-commit-config.yaml $*" + ;; + + hook-impl) + PRECOMMIT_CMD="$1" + CLEANUP=1 + shift + __init + docker run --rm \ + --entrypoint=/usr/pre-commit-venv/bin/pre-commit \ + --volume "${WORKSPACE}":/src/workspace \ + -v /var/run/docker.sock:/var/run/docker.sock \ + --env "${SKIP_ENV}" \ + --workdir /src/workspace \ + "${IMAGE_TAGGED}" \ + "${PRECOMMIT_CMD}" --config ./.pre-commit-config.yaml "$@" + ;; + + *) + __init + CLEANUP=1 + for HOOK in "$@"; do + # shellcheck disable=SC2086 + "${TOOLS_DIR}"/cbuild ${CBUILD_COMMON_ARGS} --env "${SKIP_ENV}" --cmd "/usr/pre-commit-venv/bin/pre-commit run --config ./.pre-commit-config.yaml --all-files ${HOOK}" + done + esac +else + __init + CLEANUP=1 + # shellcheck disable=SC2086 + "${TOOLS_DIR}"/cbuild ${CBUILD_COMMON_ARGS} --env "${SKIP_ENV}" --cmd "/usr/pre-commit-venv/bin/pre-commit run --config ./.pre-commit-config.yaml --all-files" +fi diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..6c6aa7c --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file