init

.github/workflows/gosec.yml

@@ -0,0 +1,29 @@
+name: gosec
+
+# Run workflow each time code is pushed to your repository and on a schedule.
+# The scheduled workflow runs every at 00:00 on Sunday UTC time.
+on:
+  push:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      actions: read
+      contents: read
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v4
+      - name: Run Gosec Security Scanner
+        uses: securego/gosec@master
+        with:
+          # we let the report trigger content trigger a failure using the GitHub Security features.
+          args: "-no-fail -fmt sarif -out results.sarif ./..."
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          # Path to SARIF file relative to the root of the repository
+          sarif_file: results.sarif

.github/workflows/lint.yml

@@ -0,0 +1,25 @@
+name: lint
+
+on:
+  push:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  golangci:
+    name: lint
+    permissions:
+      security-events: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Set up Go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version-file: "go.mod"
+      - run: sudo apt-get update && sudo apt-get install -y libpcap0.8 libpcap0.8-dev
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
+        with:
+          args: --timeout=3m

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: test
+
+on:
+  push:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  testing:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout upstream repo
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version-file: "go.mod"
+      - run: go test ./...

.github/workflows/trivy.yml

@@ -0,0 +1,37 @@
+name: trivy
+
+on:
+  push:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      actions: read
+      contents: read
+
+    steps:
+      - name: Checkout upstream repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ github.head_ref }}
+      - id: scan
+        name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@f3d98514b056d8c71a3552e8328c225bc7f6f353 # master
+        with:
+          scan-type: "fs"
+          ignore-unfixed: true
+          format: "template"
+          template: "@/contrib/sarif.tpl"
+          output: "trivy-results.sarif"
+          exit-code: 1
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        if: failure() && steps.scan.outcome == 'failure'
+        uses: github/codeql-action/upload-sarif@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0
+        with:
+          sarif_file: "trivy-results.sarif"

.gitignore

@@ -0,0 +1 @@
+tmp

README.md

@@ -1 +1,21 @@
-# hatchery
+# hatchery
+A tool to gather data + logs from SaaS and save them to object storage
+
+![overview](https://github.com/m-mizutani/hatchery/assets/605953/0d065e1e-1b40-493b-a9c5-8215f2e1691e)
+
+## Motivation
+
+Many SaaS services offer APIs for accessing data and logs, but managing them can be challenging due to various reasons:
+
+- Audit logs are often set to expire after a few months.
+- The built-in log search console provided by the service is not user-friendly and lacks centralized functionality for searching and analysis.
+
+As a result, security administrators are required to gather logs from multiple services and store them in object storage for long-term retention and analysis. However, this process is complicated by the fact that each service has its own APIs and data formats, making it difficult to implement and maintain a tool to gather logs.
+
+Hatchery is a solution designed to address these challenges by collecting data and logs from SaaS services and storing them in object storage. This facilitates log retention and prepares the data for analysis by security administrators.
+
+For those interested in importing logs from Cloud Storage to BigQuery, please refer to [swarm](https://github.com/m-mizutani/swarm).
+
+## License
+
+Apache License 2.0

cli.go

@@ -0,0 +1,16 @@
+package hatchery
+
+import (
+	"github.com/m-mizutani/goerr"
+	"github.com/urfave/cli/v2"
+)
+
+func (h *Hatchery) CLI(argv []string) error {
+	app := &cli.App{}
+
+	if err := app.Run(argv); err != nil {
+		return goerr.Wrap(err, "failed to run CLI app")
+	}
+
+	return nil
+}

destination/cloud_storage.go

@@ -0,0 +1,25 @@
+package destination
+
+import (
+	"context"
+	"io"
+)
+
+// CloudStorage is a destination that writes data to a Google cloud storage bucket.
+type CloudStorage struct {
+	bucket string
+	prefix string
+}
+
+// NewCloudStorage creates a new CloudStorage destination.
+func NewCloudStorage(bucket, prefix string) *CloudStorage {
+	return &CloudStorage{
+		bucket: bucket,
+		prefix: prefix,
+	}
+}
+
+func (c *CloudStorage) NewWriter(ctx context.Context) (io.WriteCloser, error) {
+	// Write data to Google cloud storage bucket.
+	return nil, nil
+}

go.mod

@@ -0,0 +1,15 @@
+module github.com/m-mizutani/hatchery
+
+go 1.22.0
+
+require (
+	github.com/m-mizutani/goerr v0.1.14
+	github.com/urfave/cli/v2 v2.27.4
+)
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+)

go.sum

@@ -0,0 +1,12 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/m-mizutani/goerr v0.1.14 h1:qwJ4wGoZWiHOGX/CJFvQyLRXK49EVyhOcVKAqxS/w5Q=
+github.com/m-mizutani/goerr v0.1.14/go.mod h1:OoNepSLW5oF3dQWWZ3D2lVOTbzRsePvc6LrqhXcff5Y=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
+github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=

hatchery.go

@@ -0,0 +1,15 @@
+package hatchery
+
+type Hatchery struct {
+	pipelines map[PipelineID]*Pipeline
+}
+
+type Option func(*Hatchery)
+
+func New(opts ...Option) *Hatchery {
+	h := &Hatchery{}
+	for _, opt := range opts {
+		opt(h)
+	}
+	return h
+}

interfaces.go

@@ -0,0 +1,18 @@
+package hatchery
+
+import (
+	"context"
+	"io"
+	"time"
+)
+
+// Source is an interface that loads data from a source to a destination.
+type Source interface {
+	// Load loads data from a source to a destination. It should be called periodically to get data from the source. The interval of calling Load depends on command execution of hatchery.
+	Load(ctx context.Context, dst Destination) error
+}
+
+// Destination is an interface that writes data to data storage, messaging queue or something like that.
+type Destination interface {
+	NewWriter(ctx context.Context, ts time.Time) (io.WriteCloser, error)
+}

options.go

@@ -0,0 +1,11 @@
+package hatchery
+
+func WithPipeline(id PipelineID, src Source, dst Destination) Option {
+	return func(h *Hatchery) {
+		p := &Pipeline{
+			src: src,
+			dst: dst,
+		}
+		h.pipelines[p.ID()] = p
+	}
+}

pipeline.go

@@ -0,0 +1,13 @@
+package hatchery
+
+type PipelineID string
+
+type Pipeline struct {
+	id  PipelineID
+	src Source
+	dst Destination
+}
+
+func (p *Pipeline) ID() PipelineID {
+	return p.id
+}

pkg/types/secret.go

@@ -0,0 +1,13 @@
+package hatchery
+
+type SecretString struct {
+	v string `masq:"secret" json:"-" yaml:"-"`
+}
+
+func NewSecretString(v string) SecretString {
+	return SecretString{v: v}
+}
+
+func (s SecretString) UnsafeString() string {
+	return s.v
+}

source/context.go

@@ -0,0 +1,39 @@
+package source
+
+import (
+	"context"
+	"net/http"
+	"time"
+)
+
+type HTTPClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+type ctxHTTPClientKey struct{}
+
+func httpClientFromCtx(ctx context.Context) HTTPClient {
+	if c, ok := ctx.Value(ctxHTTPClientKey{}).(HTTPClient); ok {
+		return c
+	}
+	return http.DefaultClient
+}
+
+func InjectHTTPClient(ctx context.Context, c HTTPClient) context.Context {
+	return context.WithValue(ctx, ctxHTTPClientKey{}, c)
+}
+
+type timeFunc func() time.Time
+
+type ctxTimeFuncKey struct{}
+
+func timeFuncFromCtx(ctx context.Context) timeFunc {
+	if f, ok := ctx.Value(ctxTimeFuncKey{}).(timeFunc); ok {
+		return f
+	}
+	return time.Now
+}
+
+func InjectTimeFunc(ctx context.Context, f timeFunc) context.Context {
+	return context.WithValue(ctx, ctxTimeFuncKey{}, f)
+}