From 417aff78b54a2ec34e76913e2b39edb4e7f304f3 Mon Sep 17 00:00:00 2001 From: Maor Friedman Date: Sat, 16 Mar 2019 10:06:15 +0200 Subject: [PATCH 1/2] add gcs support --- .goreleaser.yml | 2 +- Gopkg.lock | 173 ++++++++++++++++++++++++++++++++++---- Gopkg.toml | 4 +- Makefile | 4 +- README.md | 21 ++--- cmd/cain.go | 4 +- examples/code/example.go | 2 +- examples/helm/values.yaml | 2 +- pkg/cain/cain.go | 4 +- pkg/cain/cqlsh.go | 4 +- pkg/cain/nodetool.go | 4 +- pkg/utils/kube.go | 2 +- pkg/utils/path.go | 2 +- 13 files changed, 184 insertions(+), 44 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index c766c0d..1c00cda 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -25,4 +25,4 @@ dockers: - binary: cain dockerfile: Dockerfile image_templates: - - "nuvo/{{.ProjectName}}:{{ .Tag }}" + - "maorfr/{{.ProjectName}}:{{ .Tag }}" diff --git a/Gopkg.lock b/Gopkg.lock index 8dfe7ea..c5ee622 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,22 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + digest = "1:ab38507dcaeb1aec1b6b8b425151c94ad9468812d214e546bf75d77d13f9e9c1" + name = "cloud.google.com/go" + packages = [ + "compute/metadata", + "iam", + "internal", + "internal/optional", + "internal/trace", + "internal/version", + "storage", + ] + pruneopts = "UT" + revision = "f52f9bc132541d2aa914f42100c36d10b1ef7e0c" + version = "v0.37.0" + [[projects]] digest = "1:d2ccb697dc13c8fbffafa37baae97594d5592ae8f7e113471084137315536e2b" name = "github.com/Azure/azure-pipeline-go" @@ -97,10 +113,11 @@ version = "v1.1.1" [[projects]] - digest = "1:4c0989ca0bcd10799064318923b9bc2db6b4d6338dd75f3f2d86c3511aaaf5cf" + digest = "1:5d1b5a25486fc7d4e133646d834f6fca7ba1cef9903d40e7aa786c41b89e9e91" name = "github.com/golang/protobuf" packages = [ "proto", + "protoc-gen-go/descriptor", "ptypes", "ptypes/any", "ptypes/duration", @@ -126,6 +143,14 @@ pruneopts = "UT" revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" +[[projects]] + digest = "1:f1f70abea1ab125d48396343b4c053f8fecfbdb943037bf3d29dc80c90fe60b3" + name = "github.com/googleapis/gax-go" + packages = ["v2"] + pruneopts = "UT" + revision = "beaecbbdd8af86aa3acf14180d53828ce69400b2" + version = "v2.0.4" + [[projects]] digest = "1:65c4414eeb350c47b8de71110150d0ea8a281835b1f386eacaa3ad7325929c21" name = "github.com/googleapis/gnostic" @@ -149,6 +174,14 @@ pruneopts = "UT" revision = "c63ab54fda8f77302f8d414e19933f2b6026a089" +[[projects]] + digest = "1:67474f760e9ac3799f740db2c489e6423a4cde45520673ec123ac831ad849cb8" + name = "github.com/hashicorp/golang-lru" + packages = ["simplelru"] + pruneopts = "UT" + revision = "7087cb70de9f7a8bc0a10c375cb0d2280a8edf9c" + version = "v0.5.1" + [[projects]] digest = "1:8eb1de8112c9924d59bf1d3e5c26f5eaa2bfc2a5fcbb92dc1c2e4546d695f277" name = "github.com/imdario/mergo" @@ -180,6 +213,17 @@ revision = "1624edc4454b8682399def8740d46db5e4362ba4" version = "v1.1.5" +[[projects]] + digest = "1:48addb4ab030c6885899bb73b20a86dad347a58e6fc74f20bdafab1efb26a36a" + name = "github.com/maorfr/skbn" + packages = [ + "pkg/skbn", + "pkg/utils", + ] + pruneopts = "UT" + revision = "3d8a7af7e6207a77c0532cf8c9128ef8ab1a2597" + version = "0.5.0" + [[projects]] digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" name = "github.com/modern-go/concurrent" @@ -196,17 +240,6 @@ revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" -[[projects]] - digest = "1:0afbbd82a14657dbe53e816648bd4e76e8267b9bac030851500055baca6f90c5" - name = "github.com/nuvo/skbn" - packages = [ - "pkg/skbn", - "pkg/utils", - ] - pruneopts = "UT" - revision = "f3f8aee53f67faa4d0337f7fd2406d5f265bb4f3" - version = "0.4.2" - [[projects]] branch = "master" digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" @@ -239,6 +272,29 @@ revision = "298182f68c66c05229eb03ac171abe6e309ee79a" version = "v1.0.3" +[[projects]] + digest = "1:1af1920a0f0dc25426ba2e57154b9c091ec2ed83be9107abcf83d23c6c9a4194" + name = "go.opencensus.io" + packages = [ + ".", + "exemplar", + "internal", + "internal/tagencoding", + "plugin/ochttp", + "plugin/ochttp/propagation/b3", + "stats", + "stats/internal", + "stats/view", + "tag", + "trace", + "trace/internal", + "trace/propagation", + "trace/tracestate", + ] + pruneopts = "UT" + revision = "f305e5c4e2cf345eba88de13d10de1126fa45a61" + version = "v0.19.1" + [[projects]] branch = "master" digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" @@ -249,7 +305,7 @@ [[projects]] branch = "master" - digest = "1:f7171db635740bd80e33845acf90ca909ff7ecb89967ec1272d8cd01f3f8d1d7" + digest = "1:5ad570851feb57bcc5c147ee81a54cdd023a8c66a5c716d57faed25a7b6c1dff" name = "golang.org/x/net" packages = [ "context", @@ -258,17 +314,22 @@ "http2", "http2/hpack", "idna", + "internal/timeseries", + "trace", ] pruneopts = "UT" revision = "88d92db4c548972d942ac2a3531a8a9a34c82ca6" [[projects]] branch = "master" - digest = "1:bd33fb266abe5db1fb1c04e58e98192068f05a8d22fae2a9f22270c8422bb331" + digest = "1:d71cf3891adad6f2d608396a8da36e90ac46cd51e34d5a75a31385d2682b43ce" name = "golang.org/x/oauth2" packages = [ ".", + "google", "internal", + "jws", + "jwt", ] pruneopts = "UT" revision = "f42d05182288abf10faef86d16c0d07b8d40ea2d" @@ -316,13 +377,35 @@ revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd" [[projects]] - digest = "1:08206298775e5b462e6c0333f4471b44e63f1a70e42952b6ede4ecc9572281eb" + digest = "1:4d49b6e8dcda1d1a650e7cdb9a80e075dccac4c8643ffb52251b8c21bd1a9bd9" + name = "google.golang.org/api" + packages = [ + "gensupport", + "googleapi", + "googleapi/internal/uritemplates", + "googleapi/transport", + "internal", + "iterator", + "option", + "storage/v1", + "transport/http", + "transport/http/internal/propagation", + ] + pruneopts = "UT" + revision = "e742f5a8defa1f9f5d723dfa04c962e680dc33f0" + version = "v0.2.0" + +[[projects]] + digest = "1:d2a8db567a76203e3b41c1f632d86485ffd57f8e650a0d1b19d240671c2fddd7" name = "google.golang.org/appengine" packages = [ + ".", "internal", + "internal/app_identity", "internal/base", "internal/datastore", "internal/log", + "internal/modules", "internal/remote_api", "internal/urlfetch", "urlfetch", @@ -331,6 +414,59 @@ revision = "4a4468ece617fc8205e99368fa2200e9d1fad421" version = "v1.3.0" +[[projects]] + branch = "master" + digest = "1:0e25919d4395a2626f31b9607131bed0ec6407d3579403e503eafc418c05e49f" + name = "google.golang.org/genproto" + packages = [ + "googleapis/api/annotations", + "googleapis/iam/v1", + "googleapis/rpc/code", + "googleapis/rpc/status", + ] + pruneopts = "UT" + revision = "5fe7a883aa19554f42890211544aa549836af7b7" + +[[projects]] + digest = "1:cbc746de4662c66fd24a037501bd65aa0f8ad0bfca0c92576e0abb88864e3741" + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", + "codes", + "connectivity", + "credentials", + "credentials/internal", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/binarylog", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + ] + pruneopts = "UT" + revision = "2fdaae294f38ed9a121193c51ec99fecd3b13eb7" + version = "v1.19.0" + [[projects]] digest = "1:5aebed51f26a49cb77a0c34d1328a0d29839e7a1204b14ff8e26a3df8ec61736" name = "gopkg.in/djherbis/nio.v2" @@ -445,7 +581,7 @@ revision = "2b1284ed4c93a43499e781493253e2ac5959c4fd" [[projects]] - digest = "1:92c7b523fffa5badf467754612745bfa8629c4999a44c0c2521c16e2ad50bd2a" + digest = "1:89965ed41772270efa6a201c54382ca28bba606bfb364fb6cd0c1252c57bf1a2" name = "k8s.io/client-go" packages = [ "discovery", @@ -488,8 +624,10 @@ "pkg/apis/clientauthentication/v1beta1", "pkg/version", "plugin/pkg/client/auth/exec", + "plugin/pkg/client/auth/gcp", "rest", "rest/watch", + "third_party/forked/golang/template", "tools/auth", "tools/clientcmd", "tools/clientcmd/api", @@ -506,6 +644,7 @@ "util/flowcontrol", "util/homedir", "util/integer", + "util/jsonpath", ] pruneopts = "UT" revision = "e64494209f554a6723674bd494d69445fb76a1d4" @@ -531,7 +670,7 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ - "github.com/nuvo/skbn/pkg/skbn", + "github.com/maorfr/skbn/pkg/skbn", "github.com/spf13/cobra", "k8s.io/apimachinery/pkg/apis/meta/v1", ] diff --git a/Gopkg.toml b/Gopkg.toml index 57b2b4b..5bc3d0d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,6 +1,6 @@ [[constraint]] - name = "github.com/nuvo/skbn" - version = "0.4.2" + name = "github.com/maorfr/skbn" + version = "0.5.0" [[constraint]] name = "github.com/spf13/cobra" diff --git a/Makefile b/Makefile index d6f06fc..40feaee 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ build: fmt vet # Build cain docker image docker: fmt vet cp bin/cain cain - docker build -t nuvo/cain:latest . + docker build -t maorfr/cain:latest . rm cain @@ -33,7 +33,7 @@ ifdef TRAVIS ifdef DOCKER_USER ifdef DOCKER_PASSWORD docker login -u $(DOCKER_USER) -p $(DOCKER_PASSWORD) - docker push nuvo/cain:latest + docker push maorfr/cain:latest endif endif endif diff --git a/README.md b/README.md index 6b380af..ef7d611 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -[![Release](https://img.shields.io/github/release/nuvo/cain.svg)](https://github.com/nuvo/cain/releases) -[![Travis branch](https://img.shields.io/travis/nuvo/cain/master.svg)](https://travis-ci.org/nuvo/cain) -[![Docker Pulls](https://img.shields.io/docker/pulls/nuvo/cain.svg)](https://hub.docker.com/r/nuvo/cain/) -[![Go Report Card](https://goreportcard.com/badge/github.com/nuvo/cain)](https://goreportcard.com/report/github.com/nuvo/cain) -[![license](https://img.shields.io/github/license/nuvo/cain.svg)](https://github.com/nuvo/cain/blob/master/LICENSE) +[![Release](https://img.shields.io/github/release/maorfr/cain.svg)](https://github.com/maorfr/cain/releases) +[![Travis branch](https://img.shields.io/travis/maorfr/cain/master.svg)](https://travis-ci.org/maorfr/cain) +[![Docker Pulls](https://img.shields.io/docker/pulls/maorfr/cain.svg)](https://hub.docker.com/r/maorfr/cain/) +[![Go Report Card](https://goreportcard.com/badge/github.com/maorfr/cain)](https://goreportcard.com/report/github.com/maorfr/cain) +[![license](https://img.shields.io/github/license/maorfr/cain.svg)](https://github.com/maorfr/cain/blob/master/LICENSE) # Cain @@ -25,13 +25,13 @@ Cain is now an official part of the Helm [incubator/cassandra](https://github.co ### From a release -Download the latest release from the [Releases page](https://github.com/nuvo/cain/releases) or use it with a [Docker image](https://hub.docker.com/r/nuvo/cain) +Download the latest release from the [Releases page](https://github.com/maorfr/cain/releases) or use it with a [Docker image](https://hub.docker.com/r/maorfr/cain) ### From source ``` -mkdir -p $GOPATH/src/github.com/nuvo && cd $_ -git clone https://github.com/nuvo/cain.git && cd cain +mkdir -p $GOPATH/src/github.com/maorfr && cd $_ +git clone https://github.com/maorfr/cain.git && cd cain make ``` @@ -42,7 +42,7 @@ make Cain performs a backup in the following way: 1. Backup the `keyspace` schema (using `cqlsh`). 1. Get backup data using `nodetool snapshot` - it creates a snapshot of the `keyspace` in all Cassandra pods in the given `namespace` (according to `selector`). -2. Copy the files in `parallel` to cloud storage using [Skbn](https://github.com/nuvo/skbn) - it copies the files to the specified `dst`, under `namespace//keyspace//tag/`. +2. Copy the files in `parallel` to cloud storage using [Skbn](https://github.com/maorfr/skbn) - it copies the files to the specified `dst`, under `namespace//keyspace//tag/`. 3. Clear all snapshots. #### Usage @@ -205,12 +205,13 @@ cain backup ## Support for additional storage services -Since Cain uses [Skbn](https://github.com/nuvo/skbn), adding support for additional storage services is simple. Read [this post](https://medium.com/nuvo-group-tech/copy-files-and-directories-between-kubernetes-and-s3-d290ded9a5e0) for more information. +Since Cain uses [Skbn](https://github.com/maorfr/skbn), adding support for additional storage services is simple. Read [this post](https://medium.com/maorfr-group-tech/copy-files-and-directories-between-kubernetes-and-s3-d290ded9a5e0) for more information. ## Skbn compatibility matrix | Cain version | Skbn version | |--------------|--------------| +| 0.6.0 | 0.5.0 | | 0.5.1 | 0.4.2 | | 0.5.0 | 0.4.1 | | 0.4.2 | 0.4.1 | diff --git a/cmd/cain.go b/cmd/cain.go index 6fb56e0..147ca1f 100644 --- a/cmd/cain.go +++ b/cmd/cain.go @@ -8,8 +8,8 @@ import ( "os" "strings" - "github.com/nuvo/cain/pkg/cain" - "github.com/nuvo/cain/pkg/utils" + "github.com/maorfr/cain/pkg/cain" + "github.com/maorfr/cain/pkg/utils" "github.com/spf13/cobra" ) diff --git a/examples/code/example.go b/examples/code/example.go index 9c24dea..02fb609 100644 --- a/examples/code/example.go +++ b/examples/code/example.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/nuvo/cain/pkg/cain" + "github.com/maorfr/cain/pkg/cain" ) func main() { diff --git a/examples/helm/values.yaml b/examples/helm/values.yaml index 3cc9fe2..b8c1840 100644 --- a/examples/helm/values.yaml +++ b/examples/helm/values.yaml @@ -17,7 +17,7 @@ backup: iam.amazonaws.com/role: cain # We are using kube2iam image: - repository: nuvo/cain + repository: maorfr/cain tag: latest resources: diff --git a/pkg/cain/cain.go b/pkg/cain/cain.go index 21a8f73..189e165 100644 --- a/pkg/cain/cain.go +++ b/pkg/cain/cain.go @@ -5,8 +5,8 @@ import ( "log" "path/filepath" - "github.com/nuvo/cain/pkg/utils" - "github.com/nuvo/skbn/pkg/skbn" + "github.com/maorfr/cain/pkg/utils" + "github.com/maorfr/skbn/pkg/skbn" ) // BackupOptions are the options to pass to Backup diff --git a/pkg/cain/cqlsh.go b/pkg/cain/cqlsh.go index 71ee095..588995f 100644 --- a/pkg/cain/cqlsh.go +++ b/pkg/cain/cqlsh.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strings" - "github.com/nuvo/cain/pkg/utils" - "github.com/nuvo/skbn/pkg/skbn" + "github.com/maorfr/cain/pkg/utils" + "github.com/maorfr/skbn/pkg/skbn" ) // BackupKeyspaceSchema gets the schema of the keyspace and backs it up diff --git a/pkg/cain/nodetool.go b/pkg/cain/nodetool.go index 8c1e2b5..814a973 100644 --- a/pkg/cain/nodetool.go +++ b/pkg/cain/nodetool.go @@ -6,8 +6,8 @@ import ( "log" "strings" - "github.com/nuvo/cain/pkg/utils" - "github.com/nuvo/skbn/pkg/skbn" + "github.com/maorfr/cain/pkg/utils" + "github.com/maorfr/skbn/pkg/skbn" ) // TakeSnapshots takes a snapshot using nodetool in all pods in parallel diff --git a/pkg/utils/kube.go b/pkg/utils/kube.go index fc35d54..e0f9eae 100644 --- a/pkg/utils/kube.go +++ b/pkg/utils/kube.go @@ -3,7 +3,7 @@ package utils import ( "fmt" - "github.com/nuvo/skbn/pkg/skbn" + "github.com/maorfr/skbn/pkg/skbn" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 55e256f..6e6b9f7 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -5,7 +5,7 @@ import ( "path/filepath" "strings" - "github.com/nuvo/skbn/pkg/skbn" + "github.com/maorfr/skbn/pkg/skbn" ) // GetFromAndToPathsFromK8s aggregates paths from all pods From 3338dd1cac44215c388c848f0b76d75e7ede00fb Mon Sep 17 00:00:00 2001 From: Maor Friedman Date: Sat, 16 Mar 2019 10:21:46 +0200 Subject: [PATCH 2/2] update documentation for gcs support --- README.md | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ef7d611..4cb7b0d 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Cain supports the following cloud storage services: * AWS S3 * Minio S3 * Azure Blob Storage +* Google Cloud Storage Cain is now an official part of the Helm [incubator/cassandra](https://github.com/helm/charts/tree/master/incubator/cassandra) chart! @@ -87,6 +88,16 @@ cain backup \ --dst abs://my-account/db-backup-container/cassandra ``` +Backup to Google Cloud Storage + +``` +cain backup \ + -n default \ + -l release=cassandra \ + -k keyspace \ + --dst gcs://db-backup/cassandra +``` + ### Restore Cassandra backup from cloud storage Cain performs a restore in the following way: @@ -142,6 +153,17 @@ cain restore \ -t 20180903091624 ``` +Restore from Google Cloud Storage + +``` +cain restore \ + --src gcs://db-backup/cassandra/default/ring01 + -n default \ + -k keyspace \ + -l release=cassandra \ + -t 20180903091624 +``` + ### Describe keyspace schema Cain describes the `keyspace` schema using `cqlsh`. It can return the schema itself, or a checksum of the schema file (used by `backup` and `restore`). @@ -227,18 +249,23 @@ Since Cain uses [Skbn](https://github.com/maorfr/skbn), adding support for addit ### Kubernetes Cain tries to get credentials in the following order: -1. if `KUBECONFIG` environment variable is set - skbn will use the current context from that config file -2. if `~/.kube/config` exists - skbn will use the current context from that config file with an [out-of-cluster client configuration](https://github.com/kubernetes/client-go/tree/master/examples/out-of-cluster-client-configuration) -3. if `~/.kube/config` does not exist - skbn will assume it is working from inside a pod and will use an [in-cluster client configuration](https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration) +1. if `KUBECONFIG` environment variable is set - cain will use the current context from that config file +2. if `~/.kube/config` exists - cain will use the current context from that config file with an [out-of-cluster client configuration](https://github.com/kubernetes/client-go/tree/master/examples/out-of-cluster-client-configuration) +3. if `~/.kube/config` does not exist - cain will assume it is working from inside a pod and will use an [in-cluster client configuration](https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration) ### AWS -Skbn uses the default AWS [credentials chain](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). +Cain uses the default AWS [credentials chain](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). ### Azure Blob Storage -Skbn uses `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_ACCESS_KEY` environment variables for authentication. +Cain uses `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_ACCESS_KEY` environment variables for authentication. + +### Google Cloud Storage + +Cain uses Google [Application Default Credentials](https://cloud.google.com/docs/authentication/production). +Basically, it will first look for the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. If it is not defined, it will look for the default service account, or throw an error if none is configured. ## Examples