Merge pull request #2237 from weaveworks/release/1.13.2

Release Flux 1.13.2

.circleci/config.yml

@@ -2,7 +2,8 @@ version: 2
 jobs:
   build:
     working_directory: ~/flux
-    machine: true
+    machine:
+      image: ubuntu-1604:201903-01
     environment:
       GO_VERSION: 1.12.5
       # We don't need a GOPATH but CircleCI defines it, so we override it
@@ -22,10 +23,8 @@ jobs:
       - run:
           name: Update packages and Start Memcached
           command: |
-            # These repos fail and we don't need them:
-            sudo rm /etc/apt/sources.list.d/circleci_trusty.list /etc/apt/sources.list.d/google-chrome.list
             sudo apt-get update
-            sudo apt-get install -y git rng-tools docker-ce memcached
+            sudo apt-get install -y git rng-tools memcached
             git version
             docker version
       - restore_cache:
@@ -61,13 +60,22 @@ jobs:
           name: Maybe push prerelease images
           command: |
             if [ -z "${CIRCLE_TAG}" -a "${CIRCLE_BRANCH}" == "master" ]; then
+              # Push to weaveworks org
               echo "$DOCKER_REGISTRY_PASSWORD" | docker login --username "$DOCKER_REGISTRY_USER" --password-stdin
 
-              docker tag "docker.io/weaveworks/flux:$(docker/image-tag)" "docker.io/weaveworks/flux-prerelease:$(docker/image-tag)"
+              docker tag "docker.io/fluxcd/flux:$(docker/image-tag)" "docker.io/weaveworks/flux-prerelease:$(docker/image-tag)"
               docker push "docker.io/weaveworks/flux-prerelease:$(docker/image-tag)"
 
-              docker tag "docker.io/weaveworks/helm-operator:$(docker/image-tag)" "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)"
+              docker tag "docker.io/fluxcd/helm-operator:$(docker/image-tag)" "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)"
               docker push "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)"
+
+              # Push to fluxcd org
+              echo "$DOCKER_FLUXCD_PASSWORD" | docker login --username "$DOCKER_FLUXCD_USER" --password-stdin
+              docker tag "docker.io/fluxcd/flux:$(docker/image-tag)" "docker.io/fluxcd/flux-prerelease:$(docker/image-tag)"
+              docker push "docker.io/fluxcd/flux-prerelease:$(docker/image-tag)"
+
+              docker tag "docker.io/fluxcd/helm-operator:$(docker/image-tag)" "docker.io/fluxcd/helm-operator-prerelease:$(docker/image-tag)"
+              docker push "docker.io/fluxcd/helm-operator-prerelease:$(docker/image-tag)"
             fi
       - deploy:
           name: Maybe push release image and upload binaries
@@ -76,13 +84,26 @@ jobs:
               go get github.com/weaveworks/github-release
               make release-bins
               bin/upload-binaries
+              # Push to weaveworks org
               echo "$DOCKER_REGISTRY_PASSWORD" | docker login --username "$DOCKER_REGISTRY_USER" --password-stdin
+              docker tag "docker.io/fluxcd/flux:${CIRCLE_TAG}" "docker.io/weaveworks/flux:${CIRCLE_TAG}"
               docker push "docker.io/weaveworks/flux:${CIRCLE_TAG}"
+
+              # Push to fluxcd org
+              echo "$DOCKER_FLUXCD_PASSWORD" | docker login --username "$DOCKER_FLUXCD_USER" --password-stdin
+              docker push "docker.io/fluxcd/flux:${CIRCLE_TAG}"
             fi
             if echo "${CIRCLE_TAG}" | grep -Eq "helm-[0-9]+(\.[0-9]+)*(-[a-z]+)?$"; then
-              echo "$DOCKER_REGISTRY_PASSWORD" | docker login --username "$DOCKER_REGISTRY_USER" --password-stdin
               RELEASE_TAG=$(echo "$CIRCLE_TAG" | cut -c 6-)
+
+              # Push to weaveworks org
+              echo "$DOCKER_REGISTRY_PASSWORD" | docker login --username "$DOCKER_REGISTRY_USER" --password-stdin
+              docker tag "docker.io/fluxcd/helm-operator:${RELEASE_TAG}" "docker.io/weaveworks/helm-operator:${RELEASE_TAG}"
               docker push "docker.io/weaveworks/helm-operator:${RELEASE_TAG}"
+
+              # Push to fluxcd org
+              echo "$DOCKER_FLUXCD_PASSWORD" | docker login --username "$DOCKER_FLUXCD_USER" --password-stdin
+              docker push "docker.io/fluxcd/helm-operator:${RELEASE_TAG}"
             fi
 
 workflows:

CHANGELOG.md

@@ -1,6 +1,53 @@
 This is the changelog for the Flux daemon; the changelog for the Helm
 operator is in [./CHANGELOG-helmop.md](./CHANGELOG-helmop.md).
 
+## 1.13.2 (2019-07-10)
+
+This is a patch release, including a fix for [problems with using image
+labels as timestamps][weaveworks/flux#2176].
+
+### Fixes
+
+- Because image labels are inherited from base images, fluxd cannot
+  indiscriminately use labels to determine the image created date. You
+  must now explicitly allow that behaviour with the argument
+  `--registry-use-labels` [weaveworks/flux#2176][]
+- Image timestamps can be missing (or zero) if ordering them by semver
+  version rather than timestamp [weaveworks/flux#2175][]
+- Environment variables needed by the Google Cloud SDK helper are now
+  propagated to git [weaveworks/flux#2222][]
+
+### Maintenance and documentation
+
+- Image builds are pushed to both weaveworks/ and fluxcd/ orgs on
+  DockerHub, in preparation for the project moving organisations
+  [weaveworks/flux#2213][]
+- Calculate Go dependencies more efficiently during the build
+  [weaveworks/flux#2207][]
+- Refactor to remove a spurious top-level package
+  [weaveworks/flux#2201][]
+- Update the version of Kubernetes-in-Docker used in end-to-end test,
+  to v0.4.0 [weaveworks/flux#2202][]
+- Bump the Ubuntu version used in CI [weaveworks/flux#2195][]
+
+### Thanks
+
+Thanks go to the following for contributions: @2opremio, @4c74356b41,
+@ArchiFleKs, @adrian, @alanjcastonguay, @alexanderbuhler,
+@alexhumphreys, @bobbytables, @derrickburns, @dholbach, @dlespiau,
+@gaffneyd4, @hiddeco, @hkalsi, @hlascelles, @jaksonwkr, @jblunck,
+@jwenz723, @linuxbsdfreak, @luxas, @mpashka, @nlamot, @semyonslepov,
+@squaremo, @stefanprodan, @tegamckinney, @ysaakpr.
+
+[weaveworks/flux#2175]: https://github.com/weaveworks/flux/pull/2175
+[weaveworks/flux#2176]: https://github.com/weaveworks/flux/pull/2176
+[weaveworks/flux#2195]: https://github.com/weaveworks/flux/pull/2195
+[weaveworks/flux#2201]: https://github.com/weaveworks/flux/pull/2201
+[weaveworks/flux#2202]: https://github.com/weaveworks/flux/pull/2202
+[weaveworks/flux#2207]: https://github.com/weaveworks/flux/pull/2207
+[weaveworks/flux#2213]: https://github.com/weaveworks/flux/pull/2213
+[weaveworks/flux#2222]: https://github.com/weaveworks/flux/pull/2222
+
 ## 1.13.1 (2019-06-27)
 
 This is a patch release.

Makefile

@@ -22,11 +22,11 @@ GOBIN?=$(shell echo `go env GOPATH`/bin)
 # if you're testing out the Makefile with `-W` (pretend a file is
 # new); use the full path to the pretend-new file, e.g.,
 #  `make -W $PWD/registry/registry.go`
-godeps=$(shell go list -f '{{join .Deps "\n"}}' $1 | grep -v /vendor/ | xargs go list -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}')
+godeps=$(shell go list -deps -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}' $(1))
 
-FLUXD_DEPS:=$(call godeps,./cmd/fluxd)
-FLUXCTL_DEPS:=$(call godeps,./cmd/fluxctl)
-HELM_OPERATOR_DEPS:=$(call godeps,./cmd/helm-operator)
+FLUXD_DEPS:=$(call godeps,./cmd/fluxd/...)
+FLUXCTL_DEPS:=$(call godeps,./cmd/fluxctl/...)
+HELM_OPERATOR_DEPS:=$(call godeps,./cmd/helm-operator/...)
 
 IMAGE_TAG:=$(shell ./docker/image-tag)
 VCS_REF:=$(shell git rev-parse HEAD)
@@ -63,7 +63,7 @@ e2e: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done
 build/.%.done: docker/Dockerfile.%
 	mkdir -p ./build/docker/$*
 	cp $^ ./build/docker/$*/
-	$(SUDO) docker build -t docker.io/weaveworks/$* -t docker.io/weaveworks/$*:$(IMAGE_TAG) \
+	$(SUDO) docker build -t docker.io/fluxcd/$* -t docker.io/fluxcd/$*:$(IMAGE_TAG) \
 		--build-arg VCS_REF="$(VCS_REF)" \
 		--build-arg BUILD_DATE="$(BUILD_DATE)" \
 		-f build/docker/$*/Dockerfile.$* ./build/docker/$*
@@ -113,15 +113,12 @@ cache/%/helm-$(HELM_VERSION): docker/helm.version
 	mv cache/$*/helm $@
 
 $(GOBIN)/fluxctl: $(FLUXCTL_DEPS)
-$(GOBIN)/fluxctl: ./cmd/fluxctl/*.go
 	go install ./cmd/fluxctl
 
 $(GOBIN)/fluxd: $(FLUXD_DEPS)
-$(GOBIN)/fluxd: cmd/fluxd/*.go
 	go install ./cmd/fluxd
 
 $(GOBIN)/helm-operator: $(HELM_OPERATOR_DEPS)
-$(GOBIN)/help-operator: cmd/helm-operator/*.go
 	go install ./cmd/helm-operator
 
 integration-test: all

api/v11/api.go

@@ -4,14 +4,14 @@ package v11
 import (
 	"context"
 
-	"github.com/weaveworks/flux"
 	"github.com/weaveworks/flux/api/v10"
 	"github.com/weaveworks/flux/api/v6"
+	"github.com/weaveworks/flux/resource"
 )
 
 type ListServicesOptions struct {
 	Namespace string
-	Services  []flux.ResourceID
+	Services  []resource.ID
 }
 
 type Server interface {

api/v6/api.go

@@ -3,16 +3,16 @@ package v6
 import (
 	"context"
 
-	"github.com/weaveworks/flux"
 	"github.com/weaveworks/flux/cluster"
 	"github.com/weaveworks/flux/git"
 	"github.com/weaveworks/flux/job"
+	"github.com/weaveworks/flux/resource"
 	"github.com/weaveworks/flux/ssh"
 	"github.com/weaveworks/flux/update"
 )
 
 type ImageStatus struct {
-	ID         flux.ResourceID
+	ID         resource.ID
 	Containers []Container
 }
 
@@ -31,13 +31,13 @@ const (
 )
 
 type ControllerStatus struct {
-	ID         flux.ResourceID
+	ID         resource.ID
 	Containers []Container
 	ReadOnly   ReadOnlyReason
 	Status     string
 	Rollout    cluster.RolloutStatus
 	SyncError  string
-	Antecedent flux.ResourceID
+	Antecedent resource.ID
 	Labels     map[string]string
 	Automated  bool
 	Locked     bool

chart/flux/README.md

@@ -232,6 +232,7 @@ The following tables lists the configurable parameters of the Weave Flux chart a
 | `registry.insecureHosts`                          | `None`                                               | Use HTTP rather than HTTPS for the image registry domains
 | `registry.cacheExpiry`                            | `None`                                               | Duration to keep cached image info (deprecated)
 | `registry.excludeImage`                           | `None`                                               | Do not scan images that match these glob expressions; if empty, 'k8s.gcr.io/*' images are excluded
+| `registry.useTimestampLabels`                     | `None`                                               | Allow usage of (RFC3339) timestamp labels from (canonical) image refs that match these glob expressions; if empty, 'index.docker.io/weaveworks/*' images are allowed
 | `registry.ecr.region`                             | `None`                                               | Restrict ECR scanning to these AWS regions; if empty, only the cluster's region will be scanned
 | `registry.ecr.includeId`                          | `None`                                               | Restrict ECR scanning to these AWS account IDs; if empty, all account IDs that aren't excluded may be scanned
 | `registry.ecr.excludeId`                          | `602401143452`                                       | Do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account

chart/flux/templates/deployment.yaml

@@ -196,6 +196,9 @@ spec:
           {{- if .Values.registry.excludeImage }}
           - --registry-exclude-image={{ .Values.registry.excludeImage }}
           {{- end }}
+          {{- if .Values.registry.useTimestampLabels }}
+          - --registry-use-labels={{ .Values.registry.useTimestampLabels }}
+          {{- end }}
           {{- if .Values.registry.ecr.region }}
           - --registry-ecr-region={{ .Values.registry.ecr.region }}
           {{- end }}

chart/flux/values.yaml

@@ -9,7 +9,7 @@ logFormat: fmt
 
 image:
   repository: docker.io/weaveworks/flux
-  tag: 1.13.1
+  tag: 1.13.2
   pullPolicy: IfNotPresent
   pullSecret:
 
@@ -177,6 +177,8 @@ registry:
   cacheExpiry:
   # Do not scan images that match these glob expressions
   excludeImage:
+  # Allow usage of (RFC3339) timestamp labels from (canonical) image refs that match these glob expressions
+  useTimestampLabels:
   # AWS ECR settings
   ecr:
     region:

cluster/cluster.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 
-	"github.com/weaveworks/flux"
 	"github.com/weaveworks/flux/policy"
 	"github.com/weaveworks/flux/resource"
 	"github.com/weaveworks/flux/ssh"
@@ -27,8 +26,8 @@ const (
 type Cluster interface {
 	// Get all of the services (optionally, from a specific namespace), excluding those
 	AllWorkloads(ctx context.Context, maybeNamespace string) ([]Workload, error)
-	SomeWorkloads(ctx context.Context, ids []flux.ResourceID) ([]Workload, error)
-	IsAllowedResource(flux.ResourceID) bool
+	SomeWorkloads(ctx context.Context, ids []resource.ID) ([]Workload, error)
+	IsAllowedResource(resource.ID) bool
 	Ping() error
 	Export(ctx context.Context) ([]byte, error)
 	Sync(SyncSet) error
@@ -60,7 +59,7 @@ type RolloutStatus struct {
 
 // Workload describes a cluster resource that declares versioned images.
 type Workload struct {
-	ID     flux.ResourceID
+	ID     resource.ID
 	Status string // A status summary for display
 	// Is the controller considered read-only because it's under the
 	// control of the platform. In the case of Kubernetes, we simply
@@ -70,7 +69,7 @@ type Workload struct {
 	// resource through some mechanism (like an operator, or custom
 	// resource controller), we try to record the ID of that resource
 	// in this field.
-	Antecedent flux.ResourceID
+	Antecedent resource.ID
 	Labels     map[string]string
 	Policies   policy.Set
 	Rollout    RolloutStatus

cluster/kubernetes/images.go

@@ -11,9 +11,9 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"github.com/weaveworks/flux"
 	"github.com/weaveworks/flux/image"
 	"github.com/weaveworks/flux/registry"
+	"github.com/weaveworks/flux/resource"
 )
 
 func mergeCredentials(log func(...interface{}) error,
@@ -145,7 +145,7 @@ func (c *Cluster) ImagesToFetch() registry.ImageCreds {
 
 			imageCreds := make(registry.ImageCreds)
 			for _, workload := range workloads {
-				logger := log.With(c.logger, "resource", flux.MakeResourceID(ns.Name, kind, workload.GetName()))
+				logger := log.With(c.logger, "resource", resource.MakeID(ns.Name, kind, workload.GetName()))
 				mergeCredentials(logger.Log, c.includeImage, c.client, ns.Name, workload.podTemplate, imageCreds, seenCreds)
 			}
 

cluster/kubernetes/kubernetes.go

@@ -17,11 +17,11 @@ import (
 	k8sclientdynamic "k8s.io/client-go/dynamic"
 	k8sclient "k8s.io/client-go/kubernetes"
 
-	"github.com/weaveworks/flux"
 	"github.com/weaveworks/flux/cluster"
-	"github.com/weaveworks/flux/cluster/kubernetes/resource"
+	kresource "github.com/weaveworks/flux/cluster/kubernetes/resource"
 	fhrclient "github.com/weaveworks/flux/integrations/client/clientset/versioned"
 	"github.com/weaveworks/flux/ssh"
+	"github.com/weaveworks/flux/resource"
 )
 
 type coreClient k8sclient.Interface
@@ -98,7 +98,7 @@ type Cluster struct {
 
 	// syncErrors keeps a record of all per-resource errors during
 	// the sync from Git repo to the cluster.
-	syncErrors   map[flux.ResourceID]error
+	syncErrors   map[resource.ID]error
 	muSyncErrors sync.RWMutex
 
 	allowedNamespaces []string
@@ -128,7 +128,7 @@ func NewCluster(client ExtendedClient, applier Applier, sshKeyRing ssh.KeyRing,
 // SomeWorkloads returns the workloads named, missing out any that don't
 // exist in the cluster or aren't in an allowed namespace.
 // They do not necessarily have to be returned in the order requested.
-func (c *Cluster) SomeWorkloads(ctx context.Context, ids []flux.ResourceID) (res []cluster.Workload, err error) {
+func (c *Cluster) SomeWorkloads(ctx context.Context, ids []resource.ID) (res []cluster.Workload, err error) {
 	var workloads []cluster.Workload
 	for _, id := range ids {
 		if !c.IsAllowedResource(id) {
@@ -192,7 +192,7 @@ func (c *Cluster) AllWorkloads(ctx context.Context, namespace string) (res []clu
 
 			for _, workload := range workloads {
 				if !isAddon(workload) {
-					id := flux.MakeResourceID(ns.Name, kind, workload.GetName())
+					id := resource.MakeID(ns.Name, kind, workload.GetName())
 					c.muSyncErrors.RLock()
 					workload.syncError = c.syncErrors[id]
 					c.muSyncErrors.RUnlock()
@@ -208,7 +208,7 @@ func (c *Cluster) AllWorkloads(ctx context.Context, namespace string) (res []clu
 func (c *Cluster) setSyncErrors(errs cluster.SyncError) {
 	c.muSyncErrors.Lock()
 	defer c.muSyncErrors.Unlock()
-	c.syncErrors = make(map[flux.ResourceID]error)
+	c.syncErrors = make(map[resource.ID]error)
 	for _, e := range errs {
 		c.syncErrors[e.ResourceID] = e.Error
 	}
@@ -317,7 +317,7 @@ func (c *Cluster) getAllowedAndExistingNamespaces(ctx context.Context) ([]apiv1.
 	return namespaces.Items, nil
 }
 
-func (c *Cluster) IsAllowedResource(id flux.ResourceID) bool {
+func (c *Cluster) IsAllowedResource(id resource.ID) bool {
 	if len(c.allowedNamespaces) == 0 {
 		// All resources are allowed when all namespaces are allowed
 		return true
@@ -326,7 +326,7 @@ func (c *Cluster) IsAllowedResource(id flux.ResourceID) bool {
 	namespace, kind, name := id.Components()
 	namespaceToCheck := namespace
 
-	if namespace == resource.ClusterScope {
+	if namespace == kresource.ClusterScope {
 		// All cluster-scoped resources (not namespaced) are allowed ...
 		if kind != "namespace" {
 			return true

cluster/kubernetes/manifests.go

@@ -10,7 +10,6 @@ import (
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
-	"github.com/weaveworks/flux"
 	kresource "github.com/weaveworks/flux/cluster/kubernetes/resource"
 	"github.com/weaveworks/flux/image"
 	"github.com/weaveworks/flux/resource"
@@ -133,7 +132,7 @@ func (m *manifests) ParseManifest(def []byte, source string) (map[string]resourc
 	return result, nil
 }
 
-func (m *manifests) SetWorkloadContainerImage(def []byte, id flux.ResourceID, container string, image image.Ref) ([]byte, error) {
+func (m *manifests) SetWorkloadContainerImage(def []byte, id resource.ID, container string, image image.Ref) ([]byte, error) {
 	return updateWorkload(def, id, container, image)
 }