feat(subscriber): add log_subscriber to write real-time logs to redis

.github/workflows/ci.yml

@@ -9,6 +9,7 @@ jobs:
     outputs:
       ui: ${{ steps.ui.outputs.any_changed }}
       api: ${{ steps.api.outputs.any_changed }}
+      subscriber: ${{ steps.subscriber.outputs.any_changed }}
       engine: ${{ steps.engine.outputs.any_changed }}
       websocket: ${{ steps.websocket.outputs.any_changed }}
     steps:
@@ -29,6 +30,16 @@ jobs:
               .github/workflows/deploy_api_nightly.yml
               CHANGELOG.md
 
+      - name: changed files for subscriber
+        id: subscriber
+        uses: tj-actions/changed-files@v45
+        with:
+            files: |
+              server/subscriber/**
+              .github/workflows/ci.yml
+              .github/workflows/ci_subscriber.yml
+              CHANGELOG.md
+
       - name: changed files for ui
         id: ui
         uses: tj-actions/changed-files@v45
@@ -64,6 +75,10 @@ jobs:
     needs: prepare
     if: needs.prepare.outputs.api == 'true'
     uses: ./.github/workflows/ci_api.yml
+  ci-subscriber:
+    needs: prepare
+    if: needs.prepare.outputs.subscriber == 'true'
+    uses: ./.github/workflows/ci_subscriber.yml
   ci-ui:
     needs: prepare
     if: needs.prepare.outputs.ui == 'true'
@@ -80,6 +95,7 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - ci-api
+      - ci-subscriber
       - ci-ui
       - ci-websocket
       - ci-engine
@@ -181,4 +197,4 @@ jobs:
         new_tag_short: ${{ needs.ci-collect-info.outputs.new_tag_short }}
         name: ${{ needs.ci-collect-info.outputs.name }}
         sha: ${{ github.sha }}
-      secrets: inherit  
+      secrets: inherit

.github/workflows/ci_subscriber.yml

@@ -0,0 +1,43 @@
+name: ci-subscriber
+on:
+  workflow_call:
+env:
+  GO_VERSION: "1.22"
+
+jobs:
+  ci-subscriber-lint:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'v')
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: set up
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.59.1
+          args: --timeout=10m
+          working-directory: server/subscriber
+
+  ci-subscriber-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: set up
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: server/subscriber/go.sum
+      - name: test
+        run: go test ./... -v -race -coverprofile=coverage.txt -covermode=atomic -timeout 10m
+        working-directory: server/subscriber
+      - name: Send coverage report
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: subscriber

engine/Cargo.toml

@@ -18,7 +18,7 @@ homepage = "https://github.com/reearth/reearth-flow"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/reearth/reearth-flow"
 rust-version = "1.84" # Remember to update clippy.toml as well
-version = "0.0.5"
+version = "0.0.6"
 
 [profile.dev]
 opt-level = 0

engine/compose.yml

@@ -1,11 +1,16 @@
+networks:
+  reearth-flow-net:
+    external: true
 services:
   gcs:
     image: fsouza/fake-gcs-server
     ports:
       - 4443:4443
     volumes:
-      - ./tmp/gcs/data:/data # :/data/{bucket-name}
+      - ./tmp/gcs/data:/data/reearth-flow-oss-bucket # :/data/{bucket-name}
     command: --scheme http # Allow http
+    networks:
+      - reearth-flow-net
   pubsub:
     build:
       context: .
@@ -14,7 +19,7 @@ services:
     environment:
       - PROJECT_ID=local-project
       - TOPIC_IDS=flow-edge-pass-through-topic,flow-log-stream-topic,flow-job-complete-topic # Corresponding pull-type subscriptions({TOPIC}-sub) will be created automatically
-    extra_hosts: # For other docker-compose services
-      - ${DOCKER_HOST_INTERNAL:-host.docker.internal}:${DOCKER_HOST_GATEWAY:-host-gateway}
     ports:
       - 8085:8085
+    networks:
+      - reearth-flow-net

engine/plateau-gis-quality-checker/src-tauri/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 description = "Re:Earth Flow GIS Quality Checker"
 name = "plateau-gis-quality-checker"
-version = "0.0.2"
+version = "0.0.3"
 
 authors.workspace = true
 edition.workspace = true

engine/worker/README.md

@@ -9,6 +9,9 @@ cargo run --package reearth-flow-worker -- --workflow gs://reearth-flow-assets/w
 
 ### Local Development
 ``` sh
+# Create a network
+docker network create reearth-flow-net
+
 # GCS Emulator
 export STORAGE_EMULATOR_HOST=http://localhost:4443
 # Cloud Pub/Sub Emulator

go.work

@@ -1,3 +1,6 @@
 go 1.22.3
 
-use ./server/api
+use (
+    ./server/api
+    ./server/subscriber
+)