Integrate bb-browser web interface into bb-portal

Co-authored-by: Oscar Falk <oscar@meroton.com>

BUILD.bazel

@@ -9,12 +9,14 @@ load("@gazelle//:def.bzl", "gazelle")
 # gazelle:resolve proto go build/bazel/remote/execution/v2/remote_execution.proto @bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto
 # gazelle:resolve proto go google/rpc/status.proto @org_golang_google_genproto_googleapis_rpc//status
 # gazelle:resolve proto go pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth
+# gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore
 # gazelle:resolve proto go pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global
 # gazelle:resolve proto go pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc
 # gazelle:resolve proto go pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http
 # gazelle:resolve proto build/bazel/remote/execution/v2/remote_execution.proto @bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto
 # gazelle:resolve proto google/rpc/status.proto @googleapis//google/rpc:status_proto
 # gazelle:resolve proto pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth:auth_proto
+# gazelle:resolve proto pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto
 # gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto
 # gazelle:resolve proto pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc:grpc_proto
 # gazelle:resolve proto pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http:http_proto

MODULE.bazel

@@ -101,6 +101,7 @@ use_repo(
     "io_entgo_ent",
     "org_golang_google_api",
     "org_golang_google_genproto",
+    "org_golang_google_genproto_googleapis_bytestream",
     "org_golang_google_grpc",
     "org_golang_google_protobuf",
     "org_golang_x_lint",

README.md

@@ -106,6 +106,17 @@ Once you have BEP files produced by Bazel, you can upload them via the applicati
 
 BB-portal can show the same information as the web interface from BB-scheduler. To do this, you need to configure the `buildQueueStateProxy` in the portal configuration file. The interface can be found under the `Scheduler` tab in the menu.
 
+### BB-browser
+
+BB-portal can show the same information as BB-browser. Everything it can show is visible under the tab `Browser`. To make the browser functionality work, you need to configure
+
+- `actionCacheProxy`
+- `contentAddressableStorageProxy`
+- (optional) `initialSizeClassCacheProxy`
+- (optional) `fileSystemAccessCacheProxy`
+
+in the portal configuration file. Despite having the name "browser", it is not possible to browse through the content. Instead other parts of Buildbarn will generate links to the browser. To open the content in bb-portal, the the prefix for the links should be `http://url-to-bb-portal/browser/`. After the `/browser/` prefix, the rest of the URL is compatible with urls for bb-browser.
+
 ## Using GraphiQL To Explore the GraphQL API
 
 The GraphiQL explorer is available via http://localhost:8081/graphiql.

cmd/bb_portal/BUILD.bazel

@@ -14,10 +14,16 @@ go_library(
         "//ent/gen/ent/migrate",
         "//internal/api",
         "//internal/api/grpc/bes",
+        "//internal/api/grpcweb/actioncacheproxy",
         "//internal/api/grpcweb/buildqueuestateproxy",
+        "//internal/api/grpcweb/casproxy",
+        "//internal/api/grpcweb/fsacproxy",
+        "//internal/api/grpcweb/isccproxy",
+        "//internal/api/servefiles:servefiles_lib",
         "//internal/graphql",
         "//pkg/processing",
         "//pkg/proto/configuration/bb_portal",
+        "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
         "@com_github_99designs_gqlgen//graphql/handler",
         "@com_github_99designs_gqlgen//graphql/playground",
         "@com_github_buildbarn_bb_remote_execution//pkg/proto/buildqueuestate",
@@ -26,6 +32,8 @@ go_library(
         "@com_github_buildbarn_bb_storage//pkg/grpc",
         "@com_github_buildbarn_bb_storage//pkg/http",
         "@com_github_buildbarn_bb_storage//pkg/program",
+        "@com_github_buildbarn_bb_storage//pkg/proto/fsac",
+        "@com_github_buildbarn_bb_storage//pkg/proto/iscc",
         "@com_github_buildbarn_bb_storage//pkg/util",
         "@com_github_fsnotify_fsnotify//:fsnotify",
         "@com_github_gorilla_mux//:mux",
@@ -36,6 +44,7 @@ go_library(
         "@io_entgo_ent//dialect",
         "@io_entgo_ent//dialect/sql",
         "@org_golang_google_genproto//googleapis/devtools/build/v1:build",
+        "@org_golang_google_genproto_googleapis_bytestream//:bytestream",
         "@org_golang_google_grpc//:grpc",
     ],
 )

cmd/bb_portal/grpcweb_proxy_server.go

@@ -4,14 +4,22 @@ import (
 	"log"
 	"slices"
 
+	remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/actioncacheproxy"
 	"github.com/buildbarn/bb-portal/internal/api/grpcweb/buildqueuestateproxy"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/casproxy"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/fsacproxy"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/isccproxy"
 	"github.com/buildbarn/bb-portal/pkg/proto/configuration/bb_portal"
 	"github.com/buildbarn/bb-remote-execution/pkg/proto/buildqueuestate"
 	"github.com/buildbarn/bb-storage/pkg/auth"
 	bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc"
 	bb_http "github.com/buildbarn/bb-storage/pkg/http"
 	"github.com/buildbarn/bb-storage/pkg/program"
+	"github.com/buildbarn/bb-storage/pkg/proto/fsac"
+	"github.com/buildbarn/bb-storage/pkg/proto/iscc"
 	"github.com/improbable-eng/grpc-web/go/grpcweb"
+	"google.golang.org/genproto/googleapis/bytestream"
 	go_grpc "google.golang.org/grpc"
 )
 
@@ -80,4 +88,64 @@ func StartGrpcWebProxyServer(
 	} else {
 		log.Printf("Did not start BuildQueueState proxy because BuildQueueStateProxy is not configured")
 	}
+
+	if configuration.ActionCacheProxy != nil {
+		registerAndStartServer(
+			configuration.ActionCacheProxy,
+			siblingsGroup,
+			grpcClientFactory,
+			"ActionCacheProxy",
+			func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+				c := remoteexecution.NewActionCacheClient(grpcClient)
+				remoteexecution.RegisterActionCacheServer(grpcServer, actioncacheproxy.NewAcctionCacheServerImpl(c, instanceNameAuthorizer))
+			},
+		)
+	} else {
+		log.Printf("Did not start ActionCache proxy because ActionCacheProxy is not configured")
+	}
+
+	if configuration.ContentAddressableStorageProxy != nil {
+		registerAndStartServer(
+			configuration.ContentAddressableStorageProxy,
+			siblingsGroup,
+			grpcClientFactory,
+			"ContentAddressableStorageProxy",
+			func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+				c := bytestream.NewByteStreamClient(grpcClient)
+				bytestream.RegisterByteStreamServer(grpcServer, casproxy.NewCasServerImpl(c, instanceNameAuthorizer))
+			},
+		)
+	} else {
+		log.Printf("Did not start ContentAddressableStorage proxy because ContentAddressableStorageProxy is not configured")
+	}
+
+	if configuration.InitialSizeClassCacheProxy != nil {
+		registerAndStartServer(
+			configuration.InitialSizeClassCacheProxy,
+			siblingsGroup,
+			grpcClientFactory,
+			"InitialSizeClassCacheProxy",
+			func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+				c := iscc.NewInitialSizeClassCacheClient(grpcClient)
+				iscc.RegisterInitialSizeClassCacheServer(grpcServer, isccproxy.NewIsccServerImpl(c, instanceNameAuthorizer))
+			},
+		)
+	} else {
+		log.Printf("Did not start InitialSizeClassCache proxy because InitialSizeClassCacheProxy is not configured")
+	}
+
+	if configuration.FileSystemAccessCacheProxy != nil {
+		registerAndStartServer(
+			configuration.FileSystemAccessCacheProxy,
+			siblingsGroup,
+			grpcClientFactory,
+			"FileSystemAccessCacheProxy",
+			func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+				c := fsac.NewFileSystemAccessCacheClient(grpcClient)
+				fsac.RegisterFileSystemAccessCacheServer(grpcServer, fsacproxy.NewFsacServerImpl(c, instanceNameAuthorizer))
+			},
+		)
+	} else {
+		log.Printf("Did not start FileSystemAccessCache proxy because FileSystemAccessCacheProxy is not configured")
+	}
 }

cmd/bb_portal/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/buildbarn/bb-portal/ent/gen/ent/migrate"
 	"github.com/buildbarn/bb-portal/internal/api"
 	"github.com/buildbarn/bb-portal/internal/api/grpc/bes"
+	"github.com/buildbarn/bb-portal/internal/api/servefiles"
 	"github.com/buildbarn/bb-portal/internal/graphql"
 	"github.com/buildbarn/bb-portal/pkg/processing"
 	"github.com/buildbarn/bb-portal/pkg/proto/configuration/bb_portal"
@@ -117,8 +118,10 @@ func main() {
 		defer watcher.Close()
 		runWatcher(watcher, dbClient, *bepFolder, blobArchiver)
 
+		serveFileService := servefiles.NewFileServerServiceFromConfiguration(dependenciesGroup, &configuration, grpcClientFactory)
+
 		router := mux.NewRouter()
-		newPortalService(blobArchiver, dbClient, router)
+		newPortalService(blobArchiver, dbClient, serveFileService, router)
 		bb_http.NewServersFromConfigurationAndServe(
 			configuration.HttpServers,
 			bb_http.NewMetricsHandler(router, "PortalUI"),
@@ -199,13 +202,18 @@ func fatal(msg string, args ...any) {
 	os.Exit(1)
 }
 
-func newPortalService(archiver processing.BlobMultiArchiver, dbClient *ent.Client, router *mux.Router) {
+func newPortalService(archiver processing.BlobMultiArchiver, dbClient *ent.Client, serveFilesService *servefiles.FileServerService, router *mux.Router) {
 	srv := handler.NewDefaultServer(graphql.NewSchema(dbClient))
 	srv.Use(entgql.Transactioner{TxOpener: dbClient})
 
 	router.PathPrefix("/graphql").Handler(srv)
 	router.Handle("/graphiql", playground.Handler("GraphQL Playground", "/graphql"))
 	router.Handle("/api/v1/bep/upload", api.NewBEPUploadHandler(dbClient, archiver)).Methods("POST")
+	if serveFilesService != nil {
+		router.HandleFunc("/api/servefile/{instanceName:(?:.*?/)?}blobs/{digestFunction}/file/{hash}-{sizeBytes}/{name}", serveFilesService.HandleFile).Methods("GET")
+		router.HandleFunc("/api/servefile/{instanceName:(?:.*?/)?}blobs/{digestFunction}/command/{hash}-{sizeBytes}/", serveFilesService.HandleCommand).Methods("GET")
+		router.HandleFunc("/api/servefile/{instanceName:(?:.*?/)?}blobs/{digestFunction}/directory/{hash}-{sizeBytes}/", serveFilesService.HandleDirectory).Methods("GET")
+	}
 	router.PathPrefix("/").Handler(frontendServer())
 }
 

config/portal.jsonnet

@@ -4,11 +4,28 @@
 // [bb-deployments](https://github.com/buildbarn/bb-deployments), i.e. it
 // assumes that the following services are running:
 // - A Buildbarn scheduler, accessible at localhost:8984
+// - A Buildbarn frontend, accessible at localhost:8980
 
 {
+  serveFilesCasConfiguration: {
+    grpc: { address: 'localhost:8980' },
+  },
+  maximumMessageSizeBytes: 2 * 1024 * 1024,
+
   httpServers: [{
     listenAddresses: [':8081'],
-    authenticationPolicy: { allow: {} },
+    authenticationPolicy: {
+      allow: {
+        public: {
+          user: 'FooBar',
+        },
+        private: {
+          groups: ['admin'],
+          instances: ['fuse', 'testingQueue'],
+          email: 'foo@example.com',
+        },
+      },
+    },
   }],
   grpcServers: [{
     listenAddresses: [':8082'],
@@ -44,4 +61,91 @@
       },
     }],
   },
+
+  actionCacheProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9434'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
+
+  contentAddressableStorageProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9435'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
+
+  initialSizeClassCacheProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9436'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
+  fileSystemAccessCacheProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9437'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
 }

frontend/.env

@@ -1,6 +1,10 @@
 NEXT_PUBLIC_BES_BACKEND_URL=http://localhost:8081
 NEXT_PUBLIC_BES_GRPC_BACKEND_URL=grpc://localhost:8082
 NEXT_PUBLIC_BB_BUILDQUEUESTATE_GRPC_BACKEND_URL=http://localhost:9433
+NEXT_PUBLIC_BB_ACTIONCACHE_GRPC_BACKEND_URL=http://localhost:9434
+NEXT_PUBLIC_BB_CAS_GRPC_BACKEND_URL=http://localhost:9435
+NEXT_PUBLIC_BB_ISCC_GRPC_BACKEND_URL=http://localhost:9436
+NEXT_PUBLIC_BB_FSAC_GRPC_BACKEND_URL=http://localhost:9437
 NEXT_PUBLIC_BROWSER_URL=https://browser.example.com
 NEXT_PUBLIC_COMPANY_NAME="Example Co"
 NEXT_PUBLIC_COMPANY_SLACK_CHANNEL_NAME=ExampleCoBuild

frontend/scripts/grpc-client-gen.sh

@@ -19,4 +19,11 @@ generate_grpc_client() {
     ${proto_file}
 }
 
+generate_grpc_client "${PROTO_DIR}/buildbarn/auth/auth.proto"
 generate_grpc_client "${PROTO_DIR}/buildbarn/buildqueuestate/buildqueuestate.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/cas/cas.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/resourceusage/resourceusage.proto"
+generate_grpc_client "${PROTO_DIR}/google/bytestream/bytestream.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/iscc/iscc.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/fsac/fsac.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/query/query.proto"