Skip to content

Commit

Permalink
feat(actions): release and build workflows (#2)
Browse files Browse the repository at this point in the history
* chore(teamlog): add team logs endpoint; auth db query fn

* chore(teamlog): add selectable team db menu

* chore(teamlog): minor style changes

* chore(semvar): add cocogitto config

* chore(gitflow): build release ci yml; cog init

* chore(gitflow): build release yml

* chore(semver): cocogitto config file

* chore(cog): adjust cocogitto config

* chore(cog): mod release ci yml; add dev docs

* chore(cog): mod release ci yml; add dev docs

* chore(docs): add dev docs

* chore(docs): add dev docs

* chore(ci): fix release yml
  • Loading branch information
esteinig authored Nov 1, 2023
1 parent 8ee0bc3 commit dbd1c31
Show file tree
Hide file tree
Showing 10 changed files with 329 additions and 43 deletions.
54 changes: 54 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# release ci: build linux binaries and attach to release with SHA256

name: build

on:
workflow_dispatch:
release:
types: [ published ]

permissions:
contents: write

jobs:

build:
name: ${{ matrix.platform.os_name }} with rust ${{ matrix.toolchain }}
runs-on: ${{ matrix.platform.os }}
strategy:
fail-fast: false
matrix:
platform:
- os_name: Linux-aarch64
os: ubuntu-20.04
target: aarch64-unknown-linux-musl
bin: cerebro-${{ github.event.release.tag_name }}-linux-arm64 # semver release tag
- os_name: Linux-x86_64
os: ubuntu-20.04
target: x86_64-unknown-linux-gnu
bin: cerebro-${{ github.event.release.tag_name }}-linux-amd64 # semver release tag
toolchain:
- stable
steps:
- uses: actions/checkout@v3
- name: Build binary
uses: houseabsolute/actions-rust-cross@v0
with:
command: "build"
target: ${{ matrix.platform.target }}
toolchain: ${{ matrix.toolchain }}
args: "--locked --release"
strip: true
- name: Rename binary (linux and macos)
run: mv target/${{ matrix.platform.target }}/release/cerebro target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }}
if: matrix.platform.os_name != 'Windows-x86_64'
- name: Generate SHA-256 of uncompressed binary
run: shasum -a 256 target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }} | cut -d ' ' -f 1 > target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }}.sha256
- name: Compress binary
run: tar -cJf target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }}.tar.xz target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }}
- name: Release binary and SHA-256 checksum to GitHub
uses: softprops/action-gh-release@v1
with:
files: |
target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }}.tar.xz
target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }}.sha256
43 changes: 43 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# release ci: create cocogitto semantic version release + changelog, enforce conventional commit compliance
# executed when a pr is made on main which pulls in a release branch (deriving from main and dev fetch)
# triggers build yml when release is published, which attaches `latest` and {version} tagged release builds and sha256 checksums

name: release

on:
workflow_dispatch:
branches: main

jobs:
release:
name: Perform release
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Conventional commits check
uses: oknozor/cocogitto-action@v3
with:
check-latest-tag-only: true

- name: Cocogitto release
id: release
uses: oknozor/cocogitto-action@v3
with:
release: true
git-user: ${{ secrets.GIT_USER }}
git-user-email: ${{ secrets.GIT_EMAIL }}

- name: Generate Changelog
run: cog changelog --at ${{ steps.release.outputs.version }} -t full_hash > GITHUB_CHANGELOG.md

- name: Upload github release
uses: softprops/action-gh-release@v1
with:
body_path: GITHUB_CHANGELOG.md
tag_name: ${{ steps.release.outputs.version }}
token: ${{ secrets.GITHUB_TOKEN }}
54 changes: 18 additions & 36 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,53 +49,35 @@ Requirements for local execution:
- `Nextflow >= v23.10.04`

```bash
# pull latest from github, show help menu, use mamba local env
# pull latest from github, show help menu, use mamba envs
nextflow run -r latest esteinig/cerebro -profile mamba --help

# provision with latest cipher kmer db build, may take some time
nextflow run -r latest esteinig/cerebro -profile mamba -e cipher \
--cipher_revision latest \
--cipher_modules full \
nextflow run -r latest esteinig/cerebro -profile mamba -entry cipher \
--revision latest \
--representation full \
--outdir cipher_db/

# example 1 full classifier run: standard qc and align, assembly, kmer classifiers
# mamba envs and cipher db build using a large resource profile

# full tax profile on input read dir (pe illumina)
nextflow run -r latest esteinig/cerebro -profile mamba,large \
--db cipher_db/ \
--outdir full_test_run/ \
--fastq "fastq/*_{R1,R2}.fq.gz"

# example 2 k-mer profiling run: standard qc and kmer tax classifiers
# mamba envs and cipher kmer db build directory using a large resource profile
# default qc and tax profile on input read dir (pe illumina)
nextflow run esteinig/cerebro -r latest -profile mamba \
--fastq "fastq/*_{R1,R2}.fq.gz" \
--databases cipher_db/

# kmer tax profile on input read dir (pe illumina)
nextflow run -r latest esteinig/cerebro -profile mamba,large,kmer \
--db cipher_db/ \
--outdir kmer_test_run/ \
--fastq "fastq/*_{R1,R2}.fq.gz"
nextflow run esteinig/cerebro -r latest -profile mamba,kmer \
--fastq "fastq/*_{R1,R2}.fq.gz" \
--databases cipher_db/

# example 3 sample sheet production: uses cerebro pipeline client to create input sample sheet
# production: cerebro client to create input sample sheet
cerebro pipeline sample-sheet
--input fastq/ \
--output sample_sheet.csv \
--glob "*_{R1,R2}.fq.gz" \
--run-id prod-test

# tax profile on dample sheet input (pe illumina)
# production enables extra checks for sample and data auditing
nextflow run -r latest esteinig/cerebro -profile mamba,large \
--db cipher_db/ \
--outdir prod_test_run/ \
--sample_sheet sample_sheet.csv \
--production true
--run-id production_test \
--glob "*_{R1,R2}.fq.gz"

# with api upload on successful completion
# see api interaction for login to get api token
nextflow run -r latest esteinig/cerebro -profile mamba,large \
--db cipher_db/ \
--outdir prod_test_run/ \
# production: tax profile on sample sheet input (pe illumina)
nextflow run esteinig/cerebro -r latest -profile mamba \
--production true \
--sample_sheet sample_sheet.csv \
--production \
--cerebro.api.enabled \
Expand Down Expand Up @@ -152,7 +134,7 @@ cd cerebro_stack_local && docker-compose up

```bash
# get cerebro binary for support tasks
curl https://github.com/esteinig/cerebro/releases/download/latest/cerebro-latest-x86_64-unknown-linux-musl.tar.gz -o - | tar xf && mv cerebro-latest-x86_64-unknown-linux-musl cerebro
curl https://github.com/esteinig/cerebro/releases/download/latest/cerebro-latest-linux-amd64.tar.xz -o - | tar -xzO > cerebro

# assume `cerebro` on $PATH
cerebro --help
Expand Down
24 changes: 21 additions & 3 deletions app/src/lib/components/dashboard/UserProfileSummary.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,14 @@
import { page } from "$app/stores";
import { Role } from "$lib/utils/types";
import { getInitials } from "$lib/utils/helpers";
import { ListBox, ListBoxItem } from "@skeletonlabs/skeleton";
import { Accordion, AccordionItem, Avatar } from "@skeletonlabs/skeleton";
let selectedTeam: string;
let initials = getInitials($page.data.userData.name);
</script>

<p class="opacity-60">User profile</p>
Expand Down Expand Up @@ -81,10 +85,24 @@
<Accordion>
{#each $page.data.userTeams.sort() as team}
<AccordionItem>
<svelte:fragment slot="lead">
</svelte:fragment>
<svelte:fragment slot="lead"></svelte:fragment>
<svelte:fragment slot="summary">{team.name}</svelte:fragment>
<svelte:fragment slot="content"><span class="opacity-60 pl-4">{team.description}</span></svelte:fragment>
<svelte:fragment slot="content">
<ListBox>
{#each team.databases as db}
<ListBoxItem bind:group={selectedTeam} name="selectedTeam" value="{db.id}" active='variant-soft' rounded='rounded-token'>
<svelte:fragment slot="lead">
<div class="w-5">
<svg aria-hidden="true" fill="none" stroke="currentColor" stroke-width="1.5" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<path d="M20.25 6.375c0 2.278-3.694 4.125-8.25 4.125S3.75 8.653 3.75 6.375m16.5 0c0-2.278-3.694-4.125-8.25-4.125S3.75 4.097 3.75 6.375m16.5 0v11.25c0 2.278-3.694 4.125-8.25 4.125s-8.25-1.847-8.25-4.125V6.375m16.5 0v3.75m-16.5-3.75v3.75m16.5 0v3.75C20.25 16.153 16.556 18 12 18s-8.25-1.847-8.25-4.125v-3.75m16.5 0c0 2.278-3.694 4.125-8.25 4.125s-8.25-1.847-8.25-4.125" stroke-linecap="round" stroke-linejoin="round"></path>
</svg>
</div>
</svelte:fragment>
<div class="opacity-60 pl-2 cursor-pointer">{db.name}</div>
</ListBoxItem>
{/each}
</ListBox>
</svelte:fragment>
</AccordionItem>
{/each}
</Accordion>
Expand Down
2 changes: 2 additions & 0 deletions app/src/routes/cerebro/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import CerebroRelease from "$lib/components/dashboard/CerebroRelease.svelte";
import TeamActivity from "$lib/components/dashboard/TeamActivity.svelte";
import UserProfileSummary from "$lib/components/dashboard/UserProfileSummary.svelte";
console.log($page.data.userTeams)
</script>


Expand Down
36 changes: 36 additions & 0 deletions cog.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
skip_untracked = false
from_latest_tag = false
ignore_merge_commits = true
generate_mono_repository_global_tag = true

# cog bumps can only be made on main or release branches
branch_whitelist = [
"main",
"release/**"
]

# before the version is bumped
# build the release binary on local as check
# set the cargo project version to this release version
pre_bump_hooks = [
"cargo build --release",
"echo 'bumping from {{latest}} to {{version}}'",
"cargo set-version {{version}}"
]

post_bump_hooks = []

pre_package_bump_hooks = []
post_package_bump_hooks = []

[git_hooks]

[commit_types]

[changelog]
path = "CHANGELOG.md"
authors = []

[bump_profiles]

[packages]
94 changes: 94 additions & 0 deletions docs/development.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
# Development

Development setup:

* `Docker` and `docker compose`
* `rust >= v1.72` with `cargo-watch`, `cargo fmt`, `cargo clippy`
* `cocogitto >= v1.2.1` for convenvional commits and semantic version releases via `cog commit {feat} "{msg}" {scope}` command-line drop-in for `git commit`

Commits must use `cog commit` or manual conventional commit style - these can be squashed or rebased for a single merge commit into `dev` from the conventional commit named branch for example `feat/new_feature` or `chore/docs`. Pull requests only into `dev` never `main`. Release commits on `main` pull changes from `dev` via a `release/{version}` branc. Release action triggers the `cog bump --auto` changelog and version bump with cross compilation pipelines that attach Linux binaries on release publication.

## Setup dev environment

```bash
# get latest cerebro bin for deployment
curl https://github.com/esteinig/cerebro/releases/download/latest/cerebro-latest-Linux_x86_64.tar.xz -o - | tar -xzO > cerebro

# clone repo to track state
git clone https://github.com/esteinig/cerebro.git@latest

# docker prefix for this deployment
CEREBRO_PREFIX="dev_local"

# link deployment to the git repo and
# deploy on `dev_local` docker prefix
# with a `dev.cerebro.localhost` domain
# via the contained traefik reverse proxy
cerebro deploy \
--dev ${pwd}/cerebro
--config local.toml \
--docker-prefix $CEREBRO_PREFIX \
--outdir dev_local/ \
--http # traefik rev proxy for http://{api,app}.dev.cerebro.localhost/
--http-domain "dev.cerebro.localhost"

# enter deployment and up the stack
cd dev_local/ && docker compose up

# changes to the app ui are reflected immediately due to node dev server
# changes to the rust codebase require rebuild of the app e.g. when changing api code

# [TBD] fmt and clippy hooks are executed for rust changes

# down the containter stack and remove the stopped containers - this is recommended
# as a local reverse proxy can return docker network errors and db connections can fail
docker compose down && docker container rm $(docker ps --format "{{.Names}}" | grep "$CEREBRO_PREFIX"-)

# build flag triggers rebuild of changed components in the docker compose file
docker compose up --build

# checkout current feat branch and commit changes
git checkout feat/new_feature
cog commit chore "minor changes to auth api" auth
git push origin feat/new_feature

# create new pr with conventional commit style
# to merge feat/new_feature into dev

# squash working commits on pr into dev to
# the primary conventional commit
#
# docs/readme branch -> dev
# readme(docs): dev section draft
#
# dev pr squash commits are published in main release changelog
```

## Git progression for release

```bash
# checkout latest main
git checkout main

# pull latest dev into main and run auto changelog
git fetch dev

# checkout auto changelog for version release
cog changelog --auto

# get update version
$RELEASE_VERSION=$(cog bump --dry-run --auto)

# checkout version release branch
git checkout -b release/$RELEASE_VERSION

# cog commit release chore to version
git add . && cog commit chore "$RELEASE_VERSION" release

# push to remote to trigger release and build actions
git push origin release/$RELEASE_VERSION

# create pr of `release/$RELEASE_VERSION` into 'main'
# release with latest changelog and binaries is created on ci/cd

```
Loading

0 comments on commit dbd1c31

Please sign in to comment.