From 0bfe88b7cd9da76e343204c4d55e047bcd876586 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Fri, 9 Feb 2024 15:45:31 +0100 Subject: [PATCH 1/2] build: publish docker images --- .github/workflows/release.yml | 102 ---------------------------------- tools/dispatch-release.mjs | 38 ------------- tools/docker.ts | 2 +- tools/docker/bake.hcl | 18 ++++++ tools/prepare-release.ts | 7 +-- tools/publish-release.ts | 32 +++++++++-- tools/utils/docker.ts | 98 ++++++++++++++++++++++++-------- tools/utils/index.ts | 13 ++--- tools/utils/options.mjs | 15 ----- 9 files changed, 130 insertions(+), 195 deletions(-) delete mode 100644 .github/workflows/release.yml delete mode 100644 tools/dispatch-release.mjs delete mode 100644 tools/utils/options.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index a9410dc28a3bcb..00000000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: release - -on: - repository_dispatch: - types: [renovate-release] - - workflow_dispatch: - inputs: - sha: - description: 'Git sha to checkout' - required: true - version: - description: 'Version to release' - required: true - tag: - description: 'Npm dist-tag' - default: 'latest' - required: false - -env: - NODE_VERSION: 18 - GIT_SHA: ${{ github.event.client_payload.sha }} - VERSION: ${{ github.event.client_payload.version }} - BUILDKIT_PROGRESS: plain - BUILDX_NO_DEFAULT_LOAD: 1 - DOCKER_PLATFORMS: linux/amd64,linux/arm64 - OWNER: ${{ github.repository_owner }} - FILE: renovate - -permissions: - contents: read - id-token: write - -jobs: - mutex: - runs-on: ubuntu-latest - - permissions: - contents: write # pushes a branch - - steps: - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0-alpha-8 - with: - branch: mutex-rel - - release-docker: - runs-on: ubuntu-latest - needs: - - mutex - - permissions: - contents: read - id-token: write - packages: write - - steps: - - name: Prepare env - run: | - if [[ "${{github.event_name}}" == "workflow_dispatch" ]]; then - echo "GIT_SHA=${{ github.event.inputs.sha }}" >> "$GITHUB_ENV" - echo "VERSION=${{ github.event.inputs.version }}" >> "$GITHUB_ENV" - fi - echo "OWNER=${OWNER,,}" >> ${GITHUB_ENV} - - - name: docker-config - uses: containerbase/internal-tools@e7bd2e8cedd99c9b24982865534cb7c9bf88620b # v3.0.55 - with: - command: docker-config - - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ env.GIT_SHA }} - show-progress: false - - - name: Setup Node.js - uses: ./.github/actions/setup-node - with: - node-version: ${{ env.NODE_VERSION }} - os: ${{ runner.os }} - - - uses: sigstore/cosign-installer@e1523de7571e31dbe865fd2e80c5c7c23ae71eb4 # v3.4.0 - - - name: Docker registry login - run: | - echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin - - - name: Build docker images ${{ env.VERSION }} - run: pnpm build:docker build --platform=${{ env.DOCKER_PLATFORMS }} --version ${{ env.VERSION }} --tries 3 - - - name: Publish docker images ${{ env.VERSION }} - if: false - run: pnpm build:docker push --platform=${{ env.DOCKER_PLATFORMS }} --version ${{ env.VERSION }} - - - name: Sign images - if: false - run: | - cosign sign --yes ghcr.io/${{ env.OWNER }}/${{ env.FILE }}:${{ env.VERSION }} - cosign sign --yes ghcr.io/${{ env.OWNER }}/${{ env.FILE }}:${{ env.VERSION }}-full - cosign sign --yes ${{ env.FILE }}/${{ env.FILE }}:${{ env.VERSION }} - cosign sign --yes ${{ env.FILE }}/${{ env.FILE }}:${{ env.VERSION }}-full diff --git a/tools/dispatch-release.mjs b/tools/dispatch-release.mjs deleted file mode 100644 index e1581c56915d25..00000000000000 --- a/tools/dispatch-release.mjs +++ /dev/null @@ -1,38 +0,0 @@ -import got from 'got'; -import { options } from './utils/options.mjs'; - -const version = options.release; -const tag = options.tag || 'latest'; -const dry = options.dryRun; - -console.log(`Dispatching version: ${version}`); - -(async () => { - if (dry) { - console.log('DRY-RUN: done.'); - return; - } - await got( - `https://api.github.com/repos/${process.env.GITHUB_REPOSITORY}/dispatches`, - { - headers: { - 'user-agent': 'Renovate release helper', - authorization: `token ${process.env.GITHUB_TOKEN}`, - }, - method: 'POST', - json: { - event_type: 'renovate-release', - // max 10 keys here, https://github.com/peter-evans/repository-dispatch#client-payload - client_payload: { - sha: process.env.GITHUB_SHA, - ref: process.env.GITHUB_REF, - version, - tag, - }, - }, - }, - ); -})().catch((e) => { - // Ignore for now - console.warn(e.toString()); -}); diff --git a/tools/docker.ts b/tools/docker.ts index 04eff0098a1a9c..42cd36647d7435 100644 --- a/tools/docker.ts +++ b/tools/docker.ts @@ -18,7 +18,7 @@ program ) .action(async (opts) => { logger.info('Building docker images ...'); - await bake('build', opts, opts.tries - 1); + await bake('build', opts); }); program diff --git a/tools/docker/bake.hcl b/tools/docker/bake.hcl index bfe6bbd1f36bc7..4168c83ddaa667 100644 --- a/tools/docker/bake.hcl +++ b/tools/docker/bake.hcl @@ -7,6 +7,12 @@ variable "FILE" { variable "RENOVATE_VERSION" { default = "unknown" } +variable "RENOVATE_MAJOR_VERSION" { + default = "" +} +variable "RENOVATE_MAJOR_MINOR_VERSION" { + default = "" +} variable "APT_HTTP_PROXY" { default = "" @@ -64,8 +70,14 @@ target "slim" { "type=registry,ref=ghcr.io/${OWNER}/docker-build-cache:${FILE}", ] tags = [ + "ghcr.io/${OWNER}/${FILE}", "ghcr.io/${OWNER}/${FILE}:${RENOVATE_VERSION}", + "${FILE}/${FILE}", "${FILE}/${FILE}:${RENOVATE_VERSION}", + notequal("", RENOVATE_MAJOR_VERSION) ? "ghcr.io/${OWNER}/${FILE}:${RENOVATE_MAJOR_VERSION}": "", + notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "ghcr.io/${OWNER}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}": "", + notequal("", RENOVATE_MAJOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_VERSION}": "", + notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}": "", ] } @@ -78,7 +90,13 @@ target "full" { ] tags = [ "ghcr.io/${OWNER}/${FILE}:${RENOVATE_VERSION}-full", + "ghcr.io/${OWNER}/${FILE}:full", + "${FILE}/${FILE}:full", "${FILE}/${FILE}:${RENOVATE_VERSION}-full", + notequal("", RENOVATE_MAJOR_VERSION) ? "ghcr.io/${OWNER}/${FILE}:${RENOVATE_MAJOR_VERSION}-full": "", + notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "ghcr.io/${OWNER}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}-full": "", + notequal("", RENOVATE_MAJOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_VERSION}-full": "", + notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}-full": "", ] } diff --git a/tools/prepare-release.ts b/tools/prepare-release.ts index d39a7952fc1cbb..a393a3971f1fcf 100644 --- a/tools/prepare-release.ts +++ b/tools/prepare-release.ts @@ -20,15 +20,14 @@ const program = new Command('pnpm release:prepare') 'delay between tries for docker build (eg. 5s, 10m, 1h)', '30s', ) - .option('--exit-on-error [boolean]', 'exit on docker error', (s) => + .option('--exit-on-error ', 'exit on docker error', (s) => s ? s !== 'false' : undefined, - ) - .option('-d, --debug', 'output docker build'); + ); void (async () => { await program.parseAsync(); const opts = program.opts(); logger.info(`Preparing v${opts.version} ...`); await generateDocs(); - await bake('build', opts, opts.tries); + await bake('build', opts); })(); diff --git a/tools/publish-release.ts b/tools/publish-release.ts index 063b3e389ea70d..07b353e727bd7d 100644 --- a/tools/publish-release.ts +++ b/tools/publish-release.ts @@ -1,7 +1,7 @@ import { Command } from 'commander'; import { logger } from '../lib/logger'; import { parseVersion } from './utils'; -import { bake } from './utils/docker'; +import { bake, sign } from './utils/docker'; process.on('unhandledRejection', (err) => { // Will print "unhandledRejection err is not defined" @@ -13,13 +13,35 @@ const program = new Command('pnpm release:prepare') .description('Build docker images') .option('--platform ', 'docker platforms to build') .option('--version ', 'version to use as tag', parseVersion) - .option('--exit-on-error', 'exit on docker error') - .option('-d, --debug', 'output docker build'); + .option('--exit-on-error ', 'exit on docker error', (s) => + s ? s !== 'false' : undefined, + ); void (async () => { await program.parseAsync(); const opts = program.opts(); logger.info(`Publishing v${opts.version}...`); - logger.info(`TODO: publish docker images`); - await bake('push-cache', opts); + const meta = await bake('push', opts); + + if (meta?.['build-slim']?.['containerimage.digest']) { + sign( + `ghcr.io/${process.env.OWNER}/${process.env.FILE}${meta['build-slim']['containerimage.digest']}`, + opts, + ); + sign( + `${process.env.FILE}/${process.env.FILE}${meta['build-slim']['containerimage.digest']}`, + opts, + ); + } + + if (meta?.['build-full']?.['containerimage.digest']) { + sign( + `ghcr.io/${process.env.OWNER}/${process.env.FILE}@${meta['build-full']['containerimage.digest']}`, + opts, + ); + sign( + `${process.env.FILE}/${process.env.FILE}@${meta['build-full']['containerimage.digest']}`, + opts, + ); + } })(); diff --git a/tools/utils/docker.ts b/tools/utils/docker.ts index 3d01d04451a3e0..87bd97b410885a 100644 --- a/tools/utils/docker.ts +++ b/tools/utils/docker.ts @@ -1,27 +1,51 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import os from 'os'; import { setTimeout } from 'timers/promises'; +import type { SemVer } from 'semver'; import { logger } from '../../lib/logger'; import { toMs } from '../../lib/util/pretty-time'; import { exec } from './exec'; const file = 'tools/docker/bake.hcl'; +const tmp = fs.mkdtemp(path.join(os.tmpdir(), 'renovate-docker-bake-')); + +export type MetaDataItem = { + 'containerimage.digest'?: string; +}; +export type MetaData = { + 'build-slim': MetaDataItem; + 'build-full': MetaDataItem; +}; export async function bake( target: string, opts: { platform?: string; - version?: string; + version?: SemVer; args?: string[]; delay?: string; exitOnError?: boolean; + tries?: number; }, - tries: number = 0, -): Promise { +): Promise { if (opts.version) { - console.log(`Using version: ${opts.version}`); - process.env.RENOVATE_VERSION = opts.version; + console.log(`Using version: ${opts.version.version}`); + process.env.RENOVATE_VERSION = opts.version.version; + process.env.RENOVATE_MAJOR_VERSION = `${opts.version.major}`; + process.env.RENOVATE_MAJOR_MINOR_VERSION = `${opts.version.major}.${opts.version.minor}`; } - const args = ['buildx', 'bake', '--file', file]; + const metadataFile = path.join(await tmp, 'metadata.json'); + const args = [ + 'buildx', + 'bake', + '--file', + file, + '--metadata-file', + metadataFile, + '--load', + ]; if (opts.platform) { console.log(`Using platform: ${opts.platform}`); @@ -35,26 +59,56 @@ export async function bake( args.push(target); - const result = exec(`docker`, args); + for (let tries = opts.tries ?? 0; tries >= 0; tries--) { + const result = exec(`docker`, args); + if (result.signal) { + logger.error(`Signal received: ${result.signal}`); + process.exit(-1); + } else if (result.status && result.status !== 0) { + if (tries > 0) { + logger.debug(`Error occured:\n ${result.stderr}`); + const delay = opts.delay ? toMs(opts.delay) : null; + if (delay) { + logger.info(`Retrying in ${opts.delay} ...`); + await setTimeout(delay); + } + } else { + logger.error(`Error occured:\n${result.stderr}`); + if (opts.exitOnError !== false) { + process.exit(result.status); + } + return null; + } + } else { + logger.debug(`${target} succeeded:\n${result.stdout || result.stderr}`); + break; + } + } + + const meta = JSON.parse(await fs.readFile(metadataFile, 'utf8')); + logger.debug({ meta }, 'metadata'); + + return meta; +} + +export function sign( + image: string, + opts: { + args?: string[]; + exitOnError?: boolean; + }, +): void { + logger.info(`Signing ${image} ...`); + const result = exec('cosign', ['sign', '--yes', image]); if (result.signal) { logger.error(`Signal received: ${result.signal}`); - process.exit(1); + process.exit(-1); } else if (result.status && result.status !== 0) { - if (tries > 0) { - logger.debug(`Error occured:\n ${result.stderr}`); - const delay = opts.delay ? toMs(opts.delay) : null; - if (delay) { - logger.info(`Retrying in ${opts.delay} ...`); - await setTimeout(delay); - } - return bake(target, opts, tries - 1); - } else { - logger.error(`Error occured:\n${result.stderr}`); - if (opts.exitOnError !== false) { - process.exit(result.status); - } + logger.error(`Error occured:\n${result.stderr}`); + if (opts.exitOnError !== false) { + process.exit(result.status); } } else { - logger.debug(`${target} succeeded:\n${result.stdout || result.stderr}`); + logger.debug(`Succeeded:\n${result.stdout || result.stderr}`); } } diff --git a/tools/utils/index.ts b/tools/utils/index.ts index 95caa256a55bd7..e0808bd11b86f1 100644 --- a/tools/utils/index.ts +++ b/tools/utils/index.ts @@ -1,4 +1,5 @@ import fs from 'fs-extra'; +import { SemVer } from 'semver'; import { logger } from '../../lib/logger'; export const newFiles = new Set(); @@ -86,14 +87,10 @@ export function parsePositiveInt(val: string | undefined): number { * * @param val */ -export function parseVersion(val: string | undefined): string | undefined { +export function parseVersion(val: string | undefined): SemVer | undefined { if (!val) { - return val; + return undefined; } - - if (!/^\d+\.\d+\.\d+(?:-.+)?$/.test(val)) { - throw new Error(`Invalid version: ${val}`); - } - - return val; + // can throw + return new SemVer(val); } diff --git a/tools/utils/options.mjs b/tools/utils/options.mjs deleted file mode 100644 index afbadf386b0ce4..00000000000000 --- a/tools/utils/options.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import { Command } from 'commander'; - -const program = new Command(); -program - .version('0.0.1') - .requiredOption('-r, --release ', 'Version to use') - .option('-s, --sha ', 'Git sha to use') - .option('-t, --tag ', 'Npm dist-tag to publish to') - .option('-d, --dry-run'); - -program.parse(process.argv); - -export const options = program.opts(); - -export { program }; From faafd2eed2e16c9534d5326c8793d2607fc92026 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Fri, 9 Feb 2024 15:50:55 +0100 Subject: [PATCH 2/2] build: add slim images --- tools/docker/bake.hcl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/docker/bake.hcl b/tools/docker/bake.hcl index 4168c83ddaa667..13051bf56f2b32 100644 --- a/tools/docker/bake.hcl +++ b/tools/docker/bake.hcl @@ -78,6 +78,16 @@ target "slim" { notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "ghcr.io/${OWNER}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}": "", notequal("", RENOVATE_MAJOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_VERSION}": "", notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}": "", + + // TODO: legacy, remove on next major + "ghcr.io/${OWNER}/${FILE}-slim", + "ghcr.io/${OWNER}/${FILE}:${RENOVATE_VERSION}-slim", + "${FILE}/${FILE}-slim", + "${FILE}/${FILE}:${RENOVATE_VERSION}-slim", + notequal("", RENOVATE_MAJOR_VERSION) ? "ghcr.io/${OWNER}/${FILE}:${RENOVATE_MAJOR_VERSION}-slim": "", + notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "ghcr.io/${OWNER}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}-slim": "", + notequal("", RENOVATE_MAJOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_VERSION}-slim": "", + notequal("", RENOVATE_MAJOR_MINOR_VERSION) ? "${FILE}/${FILE}:${RENOVATE_MAJOR_MINOR_VERSION}-slim": "", ] }