Skip to content

Commit

Permalink
Refactor replica system
Browse files Browse the repository at this point in the history
  • Loading branch information
@benbjohnson
benbjohnson committed May 21, 2021
1 parent 8685e9f commit fb80bc1
Show file tree
Hide file tree
Showing 24 changed files with 4,182 additions and 2,822 deletions.
10 changes: 9 additions & 1 deletion .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ jobs:
steps:
- uses: actions/setup-go@v2
with:
go-version: '1.15'
go-version: '1.16'

- uses: actions/checkout@v2

Expand All @@ -19,3 +19,11 @@ jobs:
- name: Run unit tests
run: go test -v ./...

- name: Run s3 integration tests
run: go test -v ./s3 -integration
env:
LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }}
LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }}
LITESTREAM_S3_REGION: ${{ secrets.LITESTREAM_S3_REGION }}
LITESTREAM_S3_BUCKET: ${{ secrets.LITESTREAM_S3_BUCKET }}
22 changes: 11 additions & 11 deletions cmd/litestream/generations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
}

var db *litestream.DB
var r litestream.Replica
updatedAt := time.Now()
var r *litestream.Replica
dbUpdatedAt := time.Now()
if isURL(fs.Arg(0)) {
if *configPath != "" {
return fmt.Errorf("cannot specify a replica URL and the -config flag")
Expand Down Expand Up @@ -67,14 +67,14 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
}

// Determine last time database or WAL was updated.
if updatedAt, err = db.UpdatedAt(); err != nil {
if dbUpdatedAt, err = db.UpdatedAt(); err != nil {
return err
}
}

var replicas []litestream.Replica
var replicas []*litestream.Replica
if r != nil {
replicas = []litestream.Replica{r}
replicas = []*litestream.Replica{r}
} else {
replicas = db.Replicas
}
Expand All @@ -85,26 +85,26 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)

fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
for _, r := range replicas {
generations, err := r.Generations(ctx)
generations, err := r.Client.Generations(ctx)
if err != nil {
log.Printf("%s: cannot list generations: %s", r.Name(), err)
continue
}

// Iterate over each generation for the replica.
for _, generation := range generations {
stats, err := r.GenerationStats(ctx, generation)
createdAt, updatedAt, err := r.GenerationTimeBounds(ctx, generation)
if err != nil {
log.Printf("%s: cannot find generation stats: %s", r.Name(), err)
log.Printf("%s: cannot determine generation time bounds: %s", r.Name(), err)
continue
}

fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
r.Name(),
generation,
truncateDuration(updatedAt.Sub(stats.UpdatedAt)).String(),
stats.CreatedAt.Format(time.RFC3339),
stats.UpdatedAt.Format(time.RFC3339),
truncateDuration(dbUpdatedAt.Sub(updatedAt)).String(),
createdAt.Format(time.RFC3339),
updatedAt.Format(time.RFC3339),
)
}
}
Expand Down
113 changes: 57 additions & 56 deletions cmd/litestream/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"time"

"github.com/benbjohnson/litestream"
"github.com/benbjohnson/litestream/file"
"github.com/benbjohnson/litestream/s3"
_ "github.com/mattn/go-sqlite3"
"gopkg.in/yaml.v2"
Expand Down Expand Up @@ -272,15 +273,15 @@ func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {

// ReplicaConfig represents the configuration for a single replica in a database.
type ReplicaConfig struct {
Type string `yaml:"type"` // "file", "s3"
Name string `yaml:"name"` // name of replica, optional.
Path string `yaml:"path"`
URL string `yaml:"url"`
Retention time.Duration `yaml:"retention"`
RetentionCheckInterval time.Duration `yaml:"retention-check-interval"`
SyncInterval time.Duration `yaml:"sync-interval"` // s3 only
SnapshotInterval time.Duration `yaml:"snapshot-interval"`
ValidationInterval time.Duration `yaml:"validation-interval"`
Type string `yaml:"type"` // "file", "s3"
Name string `yaml:"name"` // name of replica, optional.
Path string `yaml:"path"`
URL string `yaml:"url"`
Retention *time.Duration `yaml:"retention"`
RetentionCheckInterval *time.Duration `yaml:"retention-check-interval"`
SyncInterval *time.Duration `yaml:"sync-interval"`
SnapshotInterval *time.Duration `yaml:"snapshot-interval"`
ValidationInterval *time.Duration `yaml:"validation-interval"`

// S3 settings
AccessKeyID string `yaml:"access-key-id"`
Expand All @@ -293,24 +294,51 @@ type ReplicaConfig struct {
}

// NewReplicaFromConfig instantiates a replica for a DB based on a config.
func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (litestream.Replica, error) {
func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Replica, err error) {
// Ensure user did not specify URL in path.
if isURL(c.Path) {
return nil, fmt.Errorf("replica path cannot be a url, please use the 'url' field instead: %s", c.Path)
}

// Build replica.
r := litestream.NewReplica(db, c.Name)
if v := c.Retention; v != nil {
r.Retention = *v
}
if v := c.RetentionCheckInterval; v != nil {
r.RetentionCheckInterval = *v
}
if v := c.SyncInterval; v != nil {
r.SyncInterval = *v
} else if c.ReplicaType() == "s3" {
r.SyncInterval = 10 * time.Second // default s3 to 10s for configs
}
if v := c.SnapshotInterval; v != nil {
r.SnapshotInterval = *v
}
if v := c.ValidationInterval; v != nil {
r.ValidationInterval = *v
}

// Build and set client on replica.
switch c.ReplicaType() {
case "file":
return newFileReplicaFromConfig(c, db)
if r.Client, err = newFileReplicaClientFromConfig(c, r); err != nil {
return nil, err
}
case "s3":
return newS3ReplicaFromConfig(c, db)
if r.Client, err = newS3ReplicaClientFromConfig(c, r); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unknown replica type in config: %q", c.Type)
}

return r, nil
}

// newFileReplicaFromConfig returns a new instance of FileReplica build from config.
func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.FileReplica, err error) {
// newFileReplicaClientFromConfig returns a new instance of file.ReplicaClient built from config.
func newFileReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *file.ReplicaClient, err error) {
// Ensure URL & path are not both specified.
if c.URL != "" && c.Path != "" {
return nil, fmt.Errorf("cannot specify url & path for file replica")
Expand All @@ -335,24 +363,13 @@ func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestrea
}

// Instantiate replica and apply time fields, if set.
r := litestream.NewFileReplica(db, c.Name, path)
if v := c.Retention; v > 0 {
r.Retention = v
}
if v := c.RetentionCheckInterval; v > 0 {
r.RetentionCheckInterval = v
}
if v := c.SnapshotInterval; v > 0 {
r.SnapshotInterval = v
}
if v := c.ValidationInterval; v > 0 {
r.ValidationInterval = v
}
return r, nil
client := file.NewReplicaClient(path)
client.Replica = r
return client, nil
}

// newS3ReplicaFromConfig returns a new instance of S3Replica build from config.
func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica, err error) {
// newS3ReplicaClientFromConfig returns a new instance of s3.ReplicaClient built from config.
func newS3ReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *s3.ReplicaClient, err error) {
// Ensure URL & constituent parts are not both specified.
if c.URL != "" && c.Path != "" {
return nil, fmt.Errorf("cannot specify url & path for s3 replica")
Expand Down Expand Up @@ -402,32 +419,16 @@ func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica,
}

// Build replica.
r := s3.NewReplica(db, c.Name)
r.AccessKeyID = c.AccessKeyID
r.SecretAccessKey = c.SecretAccessKey
r.Bucket = bucket
r.Path = path
r.Region = region
r.Endpoint = endpoint
r.ForcePathStyle = forcePathStyle
r.SkipVerify = skipVerify

if v := c.Retention; v > 0 {
r.Retention = v
}
if v := c.RetentionCheckInterval; v > 0 {
r.RetentionCheckInterval = v
}
if v := c.SyncInterval; v > 0 {
r.SyncInterval = v
}
if v := c.SnapshotInterval; v > 0 {
r.SnapshotInterval = v
}
if v := c.ValidationInterval; v > 0 {
r.ValidationInterval = v
}
return r, nil
client := s3.NewReplicaClient()
client.AccessKeyID = c.AccessKeyID
client.SecretAccessKey = c.SecretAccessKey
client.Bucket = bucket
client.Path = path
client.Region = region
client.Endpoint = endpoint
client.ForcePathStyle = forcePathStyle
client.SkipVerify = skipVerify
return client, nil
}

// applyLitestreamEnv copies "LITESTREAM" prefixed environment variables to
Expand Down
54 changes: 27 additions & 27 deletions cmd/litestream/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
"path/filepath"
"testing"

"github.com/benbjohnson/litestream"
main "github.com/benbjohnson/litestream/cmd/litestream"
"github.com/benbjohnson/litestream/file"
"github.com/benbjohnson/litestream/s3"
)

Expand Down Expand Up @@ -96,9 +96,9 @@ func TestNewFileReplicaFromConfig(t *testing.T) {
r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{Path: "/foo"}, nil)
if err != nil {
t.Fatal(err)
} else if r, ok := r.(*litestream.FileReplica); !ok {
} else if client, ok := r.Client.(*file.ReplicaClient); !ok {
t.Fatal("unexpected replica type")
} else if got, want := r.Path(), "/foo"; got != want {
} else if got, want := client.Path(), "/foo"; got != want {
t.Fatalf("Path=%s, want %s", got, want)
}
}
Expand All @@ -108,17 +108,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo/bar"}, nil)
if err != nil {
t.Fatal(err)
} else if r, ok := r.(*s3.Replica); !ok {
} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
t.Fatal("unexpected replica type")
} else if got, want := r.Bucket, "foo"; got != want {
} else if got, want := client.Bucket, "foo"; got != want {
t.Fatalf("Bucket=%s, want %s", got, want)
} else if got, want := r.Path, "bar"; got != want {
} else if got, want := client.Path, "bar"; got != want {
t.Fatalf("Path=%s, want %s", got, want)
} else if got, want := r.Region, ""; got != want {
} else if got, want := client.Region, ""; got != want {
t.Fatalf("Region=%s, want %s", got, want)
} else if got, want := r.Endpoint, ""; got != want {
} else if got, want := client.Endpoint, ""; got != want {
t.Fatalf("Endpoint=%s, want %s", got, want)
} else if got, want := r.ForcePathStyle, false; got != want {
} else if got, want := client.ForcePathStyle, false; got != want {
t.Fatalf("ForcePathStyle=%v, want %v", got, want)
}
})
Expand All @@ -127,17 +127,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.localhost:9000/bar"}, nil)
if err != nil {
t.Fatal(err)
} else if r, ok := r.(*s3.Replica); !ok {
} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
t.Fatal("unexpected replica type")
} else if got, want := r.Bucket, "foo"; got != want {
} else if got, want := client.Bucket, "foo"; got != want {
t.Fatalf("Bucket=%s, want %s", got, want)
} else if got, want := r.Path, "bar"; got != want {
} else if got, want := client.Path, "bar"; got != want {
t.Fatalf("Path=%s, want %s", got, want)
} else if got, want := r.Region, "us-east-1"; got != want {
} else if got, want := client.Region, "us-east-1"; got != want {
t.Fatalf("Region=%s, want %s", got, want)
} else if got, want := r.Endpoint, "http://localhost:9000"; got != want {
} else if got, want := client.Endpoint, "http://localhost:9000"; got != want {
t.Fatalf("Endpoint=%s, want %s", got, want)
} else if got, want := r.ForcePathStyle, true; got != want {
} else if got, want := client.ForcePathStyle, true; got != want {
t.Fatalf("ForcePathStyle=%v, want %v", got, want)
}
})
Expand All @@ -146,17 +146,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.s3.us-west-000.backblazeb2.com/bar"}, nil)
if err != nil {
t.Fatal(err)
} else if r, ok := r.(*s3.Replica); !ok {
} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
t.Fatal("unexpected replica type")
} else if got, want := r.Bucket, "foo"; got != want {
} else if got, want := client.Bucket, "foo"; got != want {
t.Fatalf("Bucket=%s, want %s", got, want)
} else if got, want := r.Path, "bar"; got != want {
} else if got, want := client.Path, "bar"; got != want {
t.Fatalf("Path=%s, want %s", got, want)
} else if got, want := r.Region, "us-west-000"; got != want {
} else if got, want := client.Region, "us-west-000"; got != want {
t.Fatalf("Region=%s, want %s", got, want)
} else if got, want := r.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
} else if got, want := client.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
t.Fatalf("Endpoint=%s, want %s", got, want)
} else if got, want := r.ForcePathStyle, true; got != want {
} else if got, want := client.ForcePathStyle, true; got != want {
t.Fatalf("ForcePathStyle=%v, want %v", got, want)
}
})
Expand All @@ -165,17 +165,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.storage.googleapis.com/bar"}, nil)
if err != nil {
t.Fatal(err)
} else if r, ok := r.(*s3.Replica); !ok {
} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
t.Fatal("unexpected replica type")
} else if got, want := r.Bucket, "foo"; got != want {
} else if got, want := client.Bucket, "foo"; got != want {
t.Fatalf("Bucket=%s, want %s", got, want)
} else if got, want := r.Path, "bar"; got != want {
} else if got, want := client.Path, "bar"; got != want {
t.Fatalf("Path=%s, want %s", got, want)
} else if got, want := r.Region, "us-east-1"; got != want {
} else if got, want := client.Region, "us-east-1"; got != want {
t.Fatalf("Region=%s, want %s", got, want)
} else if got, want := r.Endpoint, "https://storage.googleapis.com"; got != want {
} else if got, want := client.Endpoint, "https://storage.googleapis.com"; got != want {
t.Fatalf("Endpoint=%s, want %s", got, want)
} else if got, want := r.ForcePathStyle, true; got != want {
} else if got, want := client.ForcePathStyle, true; got != want {
t.Fatalf("ForcePathStyle=%v, want %v", got, want)
}
})
Expand Down
Loading

0 comments on commit fb80bc1

Please sign in to comment.