Merge pull request #15 from rancher/pretty_print_release_info

Add better support for release workflows, update docs, and some misc changes

pkg/sync/compare.go

@@ -1,6 +1,7 @@
 package sync
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -81,18 +82,19 @@ func CompareGeneratedAssets(rootFs billy.Filesystem, newCharts, newAssets, origi
 			for _, f := range fileInfos {
 				chartVersion := f.Name()
 				splitChartVersion := strings.Split(chartVersion, "-rc")
-				chartVersionWithoutRC := strings.Join(splitChartVersion[:len(splitChartVersion)-1], "")
-				latestRCSeenSoFar, ok := latestRC[chartVersionWithoutRC]
+				chartVersionWithName := fmt.Sprintf("%s/%s", chart, chartVersion)
+				chartVersionWithNameWithoutRC := fmt.Sprintf("%s/%s", chart, splitChartVersion[0])
+				latestRCSeenSoFar, ok := latestRC[chartVersionWithNameWithoutRC]
 				if !ok {
 					// First time seeing this RC
-					latestRC[chartVersionWithoutRC] = chartVersion
+					latestRC[chartVersionWithNameWithoutRC] = chartVersionWithName
 					continue
 				}
 				// Compare with existing value
-				if latestRCSeenSoFar >= chartVersion {
+				if latestRCSeenSoFar >= chartVersionWithName {
 					continue
 				}
-				latestRC[chartVersionWithoutRC] = chartVersion
+				latestRC[chartVersionWithNameWithoutRC] = chartVersionWithName
 				if err := filesystem.RemoveAll(rootFs, filepath.Join(path, latestRCSeenSoFar)); err != nil {
 					return fmt.Errorf("Failed to remove older RC %s: %s", filepath.Join(path, latestRCSeenSoFar), err)
 				}
@@ -103,7 +105,14 @@ func CompareGeneratedAssets(rootFs billy.Filesystem, newCharts, newAssets, origi
 		if err != nil {
 			return err
 		}
-		logrus.Infof("Found the following latest release candidate versions: %s", latestRC)
+		// pretty print on the console
+		prettyLatestRC, err := json.MarshalIndent(latestRC, "", " ")
+		if err != nil {
+			// Ignore error, just fall back to printing the map directly
+			logrus.Infof("Found the following latest release candidate versions: %s", latestRC)
+		} else {
+			logrus.Infof("Found the following latest release candidate versions: %s", prettyLatestRC)
+		}
 		// Export each helm chart to newChartsWithoutRC
 		err = filesystem.WalkDir(rootFs, newCharts, func(rootFs billy.Filesystem, path string, isDir bool) error {
 			// new-assets/charts/{package}/{chart}/{version}

templates/template/.gitignore

@@ -1 +1,2 @@
-bin
+bin
+*.DS_Store

templates/template/Makefile

@@ -1,8 +1,13 @@
-CHARTS_BUILD_SCRIPT_VERSION := fdf0565
-
 pull-scripts:
 	./scripts/pull-scripts
 
+{{ if (eq .Template "staging") -}}
+
+release:
+	./scripts/release-assets
+
+{{ end -}}
+
 TARGETS := prepare patch charts clean sync validate rebase docs
 
 $(TARGETS):

templates/template/README.md

@@ -1,16 +1,18 @@
-{{ if not .SyncOptions }}{{ if .ValidateOptions -}}
+{{ if (eq .Template "source") -}}
 ## Source Branch
 
-This branch contains packages that contain Packages that will be synced to another branch. See the README.md under `packages/` for more information.
+This branch contains packages that contain Packages that will be synced to another branch.
+
+See the README.md under `packages/` for more information.
 
 The following directory structure is expected:
 ```text
 package/
   <package>/
 ```
+{{- end }}
 
-{{- end }}{{ end }}
-{{- if .SyncOptions }}{{ if .ValidateOptions -}}
+{{- if (eq .Template "staging") -}}
 ## Staging Branch
 
 This branch contains generated assets that have not been officially released yet.
@@ -28,9 +30,9 @@ charts/
         # Unarchived Helm chart
   ...
 ```
+{{- end }}
 
-{{- end }}{{ end }}
-{{- if .SyncOptions }}{{ if not .ValidateOptions -}}
+{{- if (eq .Template "live") -}}
 ## Live Branch
 
 This branch contains generated assets that have been officially released on {{ .HelmRepoConfiguration.CNAME }}.
@@ -47,8 +49,7 @@ charts/
       <packageVersion>
         # Unarchived Helm chart
 ```
-
-{{- end }}{{ end }}
+{{- end }}
 
 ### Configuration
 
@@ -63,7 +64,9 @@ This branch syncs with the generated assets from the following branches:
 - {{ .Branch }} at {{ .UpstreamOptions.URL }}{{ if .DropReleaseCandidates }} (only latest assets){{ end }}
 {{- end }}
 
-To release a new version of a chart, please open the relevant PRs to one of these branches. Merging should trigger a sync workflow on pushing to these branches.
+To release a new version of a chart, please open the relevant PRs to one of these branches. 
+
+Merging should trigger a sync workflow on pushing to these branches.
 
 {{- end }}
 {{- if .ValidateOptions }}
@@ -79,24 +82,127 @@ Before submitting any PRs, a Github Workflow will check to see if your package d
 
 {{- end }}
 
+{{- if (eq .Template "source") }}
+
+### Making Changes
+
+As a developer making changes to a particular package, you will usually follow the following steps:
+
+#### If this is the first time you are adding a package:
+
+```shell
+PACKAGE=<packageName>
+mkdir -p packages/${PACKAGE}
+touch packages/${PACKAGE}/package.yaml
+```
+
+See `packages/README.md` to configure the `packages/${PACKAGE}/package.yaml` file based on the Package that you are planning to add.
+
+To make changes, see the steps listed below.
+
+#### If the package already exists
+
+If you are working with a single Package, set `export PACKAGE=<packageName>` to inform the scripts that you only want to make changes to a particular package.
+
+This will prevent the scripts from running commands on every package in this repository.
+
+You'll also want to update the `packageVersion` and `releaseCandidateVersion` located in `packages/${PACKAGE}/package.yaml`.
+
+See the section below for how to update this field.
+
+Once you have made those changes, the Workflow will be:
+```shell
+make prepare # Instantiates the chart in the workingDir specified in the package.yaml
+# Make your changes here to the workingDir directly here
+make patch # Saves changes to generated-changes/
+make clean # Cleans up your workingDir, leaving behind only the generated-changes/
+```
+
+Once your directory is clean, you are ready to submit a PR.
+
+#### Versioning Packages
+
+If this `major.minor.patch` (e.g. `0.0.1`) version of the Chart has never been released, reset the `packageVersion` to `01` and the `releaseCandidateVersion` to `00`.
+
+If this `major.minor.patch` (e.g. `0.0.1`) version of the Chart has been released before:
+- If this is the first time you are making a change to this chart for a specific Rancher release (i.e. the current `packageVersion` has already been released in the Live Branch), increment the `packageVersion` by 1 and reset the `releaseCandidateVersion` to `00`.
+- Otherwise, only increment the `releaseCandidateVersion` by 1.
+
+{{ end -}}
+
+{{- if (eq .Template "staging") }}
+
+### Cutting a Release
+
+In the Staging branch, cutting a release involves moving the contents of the `assets/` directory into `released/assets/`, deleting the contents of the `charts/` directory, and updating the `index.yaml` to point to the new location for those assets.
+
+This process is entirely automated via the `make release` command.
+
+Once you successfully run the `make release` command, ensure the following is true:
+- The `assets/` and `charts/` directories each only have a single file contained within them: `README.md`
+- The `released/assets/` directory has a .tgz file for each releaseCandidateVersion of a Chart that was created during this release.
+- The `index.yaml` and `released/assets/index.yaml` both are identical and the `index.yaml`'s diff shows only two types of changes: a timestamp update or a modification of an existing URL from `assets/*` to `released/assets/*`.
+
+No other changes are expected.
+
+Note: these steps should be taken only after following the steps to cut a release to your Live Branch.
+
+{{ end -}}
+
+{{- if (eq .Template "live") }}
+
+### Cutting a Release
+
+In the Live branch, cutting a release requires you to run the `make sync` command.
+
+This command will automatically get the latest charts / resources merged into the the branches you sync with (as indicated in this branch's `configuration.yaml`) and will fail if any of those branches try to modify already released assets.
+
+If the `make sync` command fails, you might have to manually make changes to the contents of the Staging Branch to resolve any issues.
+
+Once you successfully run the `make sync` command, the logs outputted will itemize the releaseCandidateVersions picked out from the Staging branch and make exactly two changes:
+
+1. It will update the `Chart.yaml`'s version for each chart to drop the `-rcXX` from it
+
+2. It will update the `Chart.yaml`'s annotations for each chart to drop the `-rcXX` from it only for some special annotations (note: currently, the only special annotation we track is `catalog.cattle.io/auto-install`).
+
+Once you successfully run the `make release` command, ensure the following is true:
+- The `assets/` and `charts/` directories each only have a single file contained within them: `README.md`
+- The `released/assets/` directory has a .tgz file for each releaseCandidateVersion of a Chart that was created during this release.
+- The `index.yaml` and `released/assets/index.yaml` both are identical and the `index.yaml`'s diff shows only two types of changes: a timestamp update or a modification of an existing URL from `assets/*` to `released/assets/*`.
+
+No other changes are expected.
+
+{{ end -}}
+
 ### Makefile
 
-#### Package-Level (requires packages/ to exist)
+#### Basic Commands
 
-`make prepare`: Pulls in your charts from upstream and creates a basic `generated-changes/` directory with your dependencies from upstream
+`make pull-scripts`: Pulls in the version of the `charts-build-scripts` indicated in scripts.
+
+{{- if (eq .Template "source") }}
 
-`make patch`: Updates your `generated-changes/` to reflect the difference between upstream and the current working directory of your branch. Requires prepare
+`make prepare`: Pulls in your charts from upstream and creates a basic `generated-changes/` directory with your dependencies from upstream
 
-`make charts`: Runs prepare and then exports your charts to `assets/` and `charts/` and generates or updates your `index.yaml`. Can be used for testing; a Rancher Helm Repository that points to a branch that has these directories with the `index.yaml` should be able to find and deploy working copies of your chart.
+`make patch`: Updates your `generated-changes/` to reflect the difference between upstream and the current working directory of your branch (note: this command should only be run after `make prepare`).
 
 `make clean`: Cleans up all the working directories of charts to get your repository ready for a PR
 
-#### Branch-Level (requires either packages/ or assets/ + charts/)
+#### Advanced Commands
 
-`make pull-scripts`: Pulls in the version of the `charts-build-scripts` indicated in scripts
+`make charts`: Runs `make prepare` and then exports your charts to `assets/` and `charts/` and generates or updates your `index.yaml`.
 
-`make validate`: Validates your current repository branch against all the repository branches indicated in your configuration.yaml
+{{ else }}
 
 `make sync`: Syncs the assets in your current repository with the merged contents of all of the repository branches indicated in your configuration.yaml
 
-`make update`: Pulls in the latest docs, scripts, etc. from the charts-build-scripts repository
+{{ end -}}
+
+`make validate`: Validates your current repository branch against all the repository branches indicated in your configuration.yaml
+
+`make docs`: Pulls in the latest docs, scripts, etc. from the charts-build-scripts repository
+
+{{- if (eq .Template "staging") }}
+
+`make release`: moves the contents of the `assets/` directory into `released/assets/`, deletes the contents of the `charts/` directory, and updates the `index.yaml` to point to the new location for those assets.
+{{ end -}}

templates/template/_config.yml

@@ -0,0 +1 @@
+exclude: [charts]

templates/template/released/README.md

@@ -0,0 +1,5 @@
+## Released Assets
+
+This folder contains Helm chart archives for `releaseCandidateVersions` of Helm charts contained within Packages whose `packageVersion` have already been released at {{ .HelmRepoConfiguration.CNAME }}.
+
+On cutting a release, a Release Captain should run the corresponding `make release` command on this repository to move already released assets into this directory and update the index.yaml with the new chart locations.

templates/template/scripts/release-assets

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)
+
+cd ..
+
+mkdir -p released
+
+if ! [[ -d assets ]] || ! [[ -d charts ]]; then
+    echo "No assets to move to released/"
+    exit 0
+fi
+
+cp -R assets/ released/assets
+rm released/assets/README.md
+rm -R $(ls -1 -d charts/*/) $(ls -1 -d assets/*/)
+
+helm repo index --merge ./released/assets/index.yaml --url released/assets released/assets
+cp released/assets/index.yaml index.yaml

templates/template/scripts/version

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -e
 
-CHARTS_BUILD_SCRIPT_VERSION=v0.0.3
+CHARTS_BUILD_SCRIPT_VERSION=v0.0.4

templates/update.yaml

@@ -10,9 +10,11 @@ staging:
 - .gitignore
 - scripts/pull-scripts
 - scripts/regenerate-assets
+- scripts/release-assets
 - Makefile
 - assets/README.md
 - charts/README.md
+- released/README.md
 - CNAME
 - index.html
 live:
@@ -24,4 +26,5 @@ live:
 - assets/README.md
 - charts/README.md
 - CNAME
-- index.html
+- index.html
+- _config.yml