-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
api: Integrate Docker + implement s3 store
- Loading branch information
1 parent
de746c3
commit 92cef4e
Showing
15 changed files
with
245 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| # Build stage | ||
| FROM golang:1.21-alpine3.19 as builder | ||
| WORKDIR /app | ||
| COPY . . | ||
|
|
||
| RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest | ||
|
|
||
| RUN go build -o auxstream | ||
|
|
||
| # Run stage | ||
| FROM alpine:3.19 | ||
| WORKDIR /app | ||
| COPY --from=builder /app/auxstream . | ||
| COPY --from=builder /go/bin/migrate ./migrate | ||
| COPY app.env . | ||
| COPY start.sh . | ||
| COPY db/migration ./db/migration | ||
|
|
||
| EXPOSE 5009 | ||
|
|
||
| CMD ["/app/auxstream"] | ||
| ENTRYPOINT ["/app/start.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,4 +16,8 @@ test: | |
| run: | ||
| go run main.go | ||
|
|
||
|
|
||
| build: | ||
| go build -o build/auxstream | ||
|
|
||
| .PHONY: test run createdb setup-db teardown-db | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,2 @@ | ||
| DATABASE_URL=postgresql://root:secret@localhost:5432/auxstreamdb?sslmode=disable | ||
| DATABASE_URL=postgresql://postgres:secret@postgres:5432/auxstreamdb?sslmode=disable | ||
| SESSION_STRING=test_random_string |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| version: "3.9" | ||
| services: | ||
| postgres: | ||
| restart: always | ||
| image: postgres:14-alpine | ||
| environment: | ||
| - POSTGRES_USER=postgres | ||
| - POSTGRES_PASSWORD=secret | ||
| - POSTGRES_DB=auxstreamdb | ||
| api: | ||
| build: | ||
| context: . | ||
| dockerfile: Dockerfile | ||
| ports: | ||
| - "5009:5009" | ||
| environment: | ||
| - DATABASE_URL=postgresql://postgres:secret@postgres:5432/auxstreamdb?sslmode=disable | ||
|
|
||
| restart: on-failure | ||
| depends_on: | ||
| - postgres |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| package filesystem | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "fmt" | ||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| s3API "github.com/aws/aws-sdk-go/service/s3" | ||
| "github.com/aws/aws-sdk-go/service/s3/s3manager" | ||
| "log" | ||
| "sync" | ||
| ) | ||
|
|
||
| // S3Store a FileSystem interface def over s3 storage | ||
| type S3Store struct { | ||
| session *session.Session | ||
| bucketId string | ||
| uploads int | ||
| downloads int | ||
| mu sync.Mutex | ||
| } | ||
|
|
||
| func NewS3Store(bucketId string) *S3Store { | ||
| sess := session.Must(session.NewSession()) | ||
| return &S3Store{session: sess, bucketId: bucketId} | ||
| } | ||
|
|
||
| func (s3 *S3Store) Reads() int { | ||
| return s3.downloads | ||
| } | ||
|
|
||
| func (s3 *S3Store) Writes() int { | ||
| return s3.uploads | ||
| } | ||
|
|
||
| func (s3 *S3Store) Save(raw []byte) (filename string, err error) { | ||
| if len(raw) < 1 { | ||
| return filename, fmt.Errorf("empty file") | ||
| } | ||
| filename = genFileName() | ||
|
|
||
| freader := bytes.NewReader(raw) | ||
|
|
||
| // Create an uploader with the session | ||
| uploader := s3manager.NewUploader(s3.session) | ||
|
|
||
| // Upload the file to S3. | ||
| _, err = uploader.Upload(&s3manager.UploadInput{ | ||
| Bucket: aws.String(s3.bucketId), | ||
| Key: aws.String(filename), | ||
| Body: freader, | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return "", fmt.Errorf("failed to upload file, %v", err) | ||
| } | ||
| s3.mu.Lock() | ||
| s3.uploads++ | ||
| s3.mu.Unlock() | ||
| return | ||
| } | ||
|
|
||
| func (s3 *S3Store) Read(fileName string) (file File, err error) { | ||
| // Create a downloader with the session= | ||
| downloader := s3manager.NewDownloader(s3.session) | ||
| // Create a file to write the S3 Object contents to. | ||
| lfile, err := NewFile(fileName) | ||
| file = lfile | ||
|
|
||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create file %q, %v", fileName, err) | ||
| } | ||
|
|
||
| _, err = downloader.Download(lfile, &s3API.GetObjectInput{ | ||
| Bucket: aws.String(s3.bucketId), | ||
| Key: aws.String(fileName), | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to download file, %v", err) | ||
| } | ||
|
|
||
| s3.mu.Lock() | ||
| s3.downloads++ | ||
| s3.mu.Unlock() | ||
|
|
||
| return | ||
| } | ||
|
|
||
| func (s3 *S3Store) BulkSave(buf chan<- string, listOfRaw [][]byte) { | ||
| var wg sync.WaitGroup | ||
| for _, raw := range listOfRaw { | ||
| wg.Add(1) | ||
| go func(raw []byte) { | ||
| defer wg.Done() | ||
| fileName, err := s3.Save(raw) | ||
| if err != nil { | ||
| log.Println(err) | ||
| buf <- "" | ||
| return | ||
| } | ||
| buf <- fileName | ||
| }(raw) | ||
| } | ||
| wg.Wait() | ||
| close(buf) | ||
| } | ||
|
|
||
| func (s3 *S3Store) Remove(fileName string) error { | ||
| deleteObjectInput := &s3API.DeleteObjectInput{ | ||
| Bucket: aws.String(s3.bucketId), | ||
| Key: aws.String(fileName), | ||
| } | ||
|
|
||
| s3Client := s3API.New(s3.session) | ||
|
|
||
| // Delete the object | ||
| _, err := s3Client.DeleteObject(deleteObjectInput) | ||
| if err != nil { | ||
| log.Println("Error deleting object:", err) | ||
| return err | ||
| } | ||
|
|
||
| log.Printf("Object '%s' deleted from bucket '%s'\n", fileName, s3.bucketId) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| #!/bin/sh | ||
|
|
||
| set -e | ||
|
|
||
| echo "run db migration" | ||
| /app/migrate -path /app/db/migration -database "$DATABASE_URL" -verbose up | ||
|
|
||
| echo "start app" | ||
| exec "$@" |
Oops, something went wrong.