api: build using buf

Refactor our build pipeline for the protobuf API to use buf[1] to do our code generation.

This approach has a few key advantages:

1. Simpler and more maintainable. We no longer need to maintain a complex makefile that
   gets more complex with every new .proto file.

2. Automatic checking for API breakages including JSON and wire compatibility.

3. Linting and automatic code formatting to improve code consistency and enforce best
   practices. Note: linting is currently disabled pending a larger incoming PR that will
   address current issues.

[1]: https://buf.build/docs/cli/

Signed-off-by: William Findlay <william.findlay@isovalent.com>

.github/workflows/generated-files.yaml

@@ -5,27 +5,41 @@ on:
       - main
       - v*
     paths-ignore:
-      - 'docs/**'
+      - "docs/**"
   pull_request:
     paths-ignore:
-      - 'docs/**'
+      - "docs/**"
 
 jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          fetch-depth: 0
+      - name: Lint protobuf
+        run: |
+          # TODO: enable linting once we have a chance to fix the underlying issues
+          # make -C api lint EXTRA_BUF_FLAGS="--error-format=github-actions"
+          make -C api check-breaking EXTRA_BUF_FLAGS="--error-format=github-actions" BUF_BREAKING_AGAINST_BRANCH="origin/${{ github.base_ref }}"
+          make -C api format EXTRA_BUF_FLAGS="--exit-code --error-format=github-actions"
   generated-files:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
       - name: Install Go
         uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           # renovate: datasource=golang-version depName=go
-          go-version: '1.23.5'
+          go-version: "1.23.5"
       - name: Go version
         run: go version
       - name: Validate that generated files are up to date.
         run: |
           make generate
-          make codegen
+          make codegen BUF_BREAKING_AGAINST_BRANCH="origin/${{ github.base_ref }}"
           git status
           git diff
           test -z "$(git status --porcelain)"

Makefile

@@ -3,10 +3,8 @@
 
 include Makefile.defs
 
-GO ?= go
 INSTALL = $(QUIET)install
 BINDIR ?= /usr/local/bin
-CONTAINER_ENGINE ?= docker
 DOCKER_IMAGE_TAG ?= latest
 LOCAL_CLANG ?= 0
 LOCAL_CLANG_FORMAT ?= 0

Makefile.defs

@@ -8,3 +8,11 @@ endef
 define print_help_option
 	@printf "  \033[35m%-22s\033[0m  %s\n" $(1) '$(2)'
 endef
+
+GO ?= go
+CONTAINER_ENGINE ?= docker
+
+BUF ?= buf
+BUF_BREAKING_AGAINST_BRANCH ?= origin/main
+# renovate: datasource=docker
+BUILDER_IMAGE=quay.io/cilium/cilium-builder:cd04ac813fb4763f840911c88beae99efc4aa457

api/Makefile

@@ -1,27 +1,53 @@
 # Copyright 2017-2020 Authors of Cilium
 # SPDX-License-Identifier: Apache-2.0
 
-# renovate: datasource=docker
-BUILDER_IMAGE=quay.io/cilium/cilium-builder@sha256:a2036a3f6420647e6702695dabd2ffa4d2832db45157042d0255bdc707b8e1f2
-GO ?= go
+include ../Makefile.defs
+
+EXTRA_BUF_FLAGS ?=
 
 .PHONY: all
-all: proto
+all: 
+
+# FIXME: re-enable lint on the all target once we have a chance to fix the lints
+.PHONY: __all_local
+__all_local: __format_local __check-breaking_local __proto_local #__lint_local 
+
+.PHONY: __proto_local
+__proto_local: 
+	$(BUF) generate $(EXTRA_BUF_FLAGS)
+
+.PHONY: __lint_local
+__lint_local:
+	$(BUF) lint $(EXTRA_BUF_FLAGS)
+
+.PHONY: __format_local
+__format_local:
+	$(BUF) format -w $(EXTRA_BUF_FLAGS)
 
-.PHONY: proto
-proto: v1
+.PHONY: __check-breaking_local
+__check-breaking_local:
+	$(BUF) breaking /src/api --against "$(CURDIR)/../.git#branch=$(BUF_BREAKING_AGAINST_BRANCH),subdir=api/v1" $(EXTRA_BUF_FLAGS)
+
+.PHONY: proto lint format check-breaking
+all proto lint format check-breaking:
+	$(CONTAINER_ENGINE) container run --rm \
+		--volume $(CURDIR)/..:/src \
+		--workdir /src/api \
+		--user "$(shell id -u):$(shell id -g)" \
+		$(BUILDER_IMAGE) \
+		make -C /src/api __$@_local BUF_BREAKING_AGAINST_BRANCH=$(BUF_BREAKING_AGAINST_BRANCH) 
 
-.PHONY: v1
-v1:
-	docker container run --rm \
+.PHONY: debug
+debug:
+	@echo $(CONTAINER_ENGINE) container run --rm \
 		--volume $(CURDIR)/..:/src \
+		--workdir /src/api \
 		--user "$(shell id -u):$(shell id -g)" \
 		$(BUILDER_IMAGE) \
-		make -C /src/api/v1 -f Makefile.protoc
-	./export-doc.sh ../docs/content/en/docs/reference/grpc-api.md
+		buf $(EXTRA_BUF_FLAGS)
 
 .PHONY: vendor
 vendor:
 	$(GO) mod tidy
 	$(GO) mod vendor
-	$(GO) mod verify
+	$(GO) mod verify

api/buf.gen.yaml

@@ -0,0 +1,16 @@
+version: v2
+plugins:
+  - local: protoc-gen-go
+    out: ./v1
+    opt:
+      - paths=source_relative
+  - local: protoc-gen-go-json
+    out: ./v1
+    opt:
+      - paths=source_relative
+      - orig_name=true
+  - local: protoc-gen-doc
+    out: ./v1
+    opt:
+      - markdown
+      - README.md

api/buf.yaml

@@ -0,0 +1,28 @@
+version: v2
+
+modules:
+  - path: v1
+
+lint:
+  use:
+    # See full list here: https://buf.build/docs/lint/rules/
+    # TODO: it's going to take a lot of work to document everything before enabling this.
+    # - COMMENTS
+    - STANDARD
+  enum_zero_value_suffix: UNDEF
+  except:
+    # Too late to do this without breaking.
+    - ENUM_VALUE_PREFIX
+    # Might be too late to do this without breaking.
+    - PACKAGE_VERSION_SUFFIX
+  disallow_comment_ignores: false
+
+breaking:
+  use:
+    - FILE
+    - WIRE_JSON
+  except:
+    - FILE_SAME_GO_PACKAGE
+  # Ignores packages with a last component that's one of the unstable forms
+  # (e.g. foo.bar.v1alpha foo.bar.v1beta)
+  ignore_unstable_packages: true