Skip to content

Commit

Permalink
Fix storage implementation for snapshots (#497)
Browse files Browse the repository at this point in the history 
This fixed the gc of deleted namespaces. 2 changes were needed.

- Use internal type to implement storage
- Remove `omitempty` from versioned List
  • Loading branch information
@tamalsaha
tamalsaha authored May 23, 2018
1 parent 247c016 commit 0da98a7
Show file tree
Hide file tree
Showing 6 changed files with 39 additions and 35 deletions.
1 change: 1 addition & 0 deletions apis/repositories/v1alpha1/openapi_generated.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion apis/repositories/v1alpha1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,5 +52,5 @@ type SnapshotStatus struct {
type SnapshotList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Snapshot `json:"items,omitempty"`
Items []Snapshot `json:"items"`
}
7 changes: 5 additions & 2 deletions hack/dev/setup-server.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ else
esac
fi

export STASH_NAMESPACE=stash-dev
export STASH_NAMESPACE=default
export KUBE_CA=$($ONESSL get kube-ca | $ONESSL base64)
export STASH_ENABLE_WEBHOOK = true
export STASH_ENABLE_WEBHOOK=true

while test $# -gt 0; do
case "$1" in
Expand Down Expand Up @@ -75,6 +75,9 @@ while test $# -gt 0; do
esac
done

# !!! WARNING !!! Never do this in prod cluster
kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --user=system:anonymous

cat $REPO_ROOT/hack/dev/apiregistration.yaml | envsubst | kubectl apply -f -

if [ "$STASH_ENABLE_WEBHOOK" = true ]; then
Expand Down
3 changes: 3 additions & 0 deletions openapi-spec/v2/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3215,6 +3215,9 @@
]
},
"com.github.appscode.stash.apis.repositories.v1alpha1.SnapshotList": {
"required": [
"items"
],
"properties": {
"apiVersion": {
"description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources",
Expand Down
37 changes: 17 additions & 20 deletions pkg/registry/snapshot/snapshot.go
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
package snapshot

import (
api "github.com/appscode/stash/apis/repositories/v1alpha1"
"github.com/appscode/stash/apis/stash/v1alpha1"
"github.com/appscode/stash/apis/repositories"
stash "github.com/appscode/stash/apis/stash/v1alpha1"
"github.com/appscode/stash/client/clientset/versioned"
"github.com/appscode/stash/pkg/util"
"github.com/pkg/errors"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/client-go/kubernetes"
Expand All @@ -26,7 +25,6 @@ type REST struct {
var _ rest.Getter = &REST{}
var _ rest.Lister = &REST{}
var _ rest.GracefulDeleter = &REST{}
var _ rest.GroupVersionKindProvider = &REST{}

func NewREST(config *restconfig.Config) *REST {
return &REST{
Expand All @@ -37,11 +35,7 @@ func NewREST(config *restconfig.Config) *REST {
}

func (r *REST) New() runtime.Object {
return &api.Snapshot{}
}

func (r *REST) GroupVersionKind(containingGV schema.GroupVersion) schema.GroupVersionKind {
return api.SchemeGroupVersion.WithKind(api.ResourceKindSnapshot)
return &repositories.Snapshot{}
}

func (r *REST) Get(ctx apirequest.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
Expand All @@ -63,7 +57,7 @@ func (r *REST) Get(ctx apirequest.Context, name string, options *metav1.GetOptio
return nil, errors.New("respective repository not found. error:" + err.Error())
}

snapshots := make([]api.Snapshot, 0)
snapshots := make([]repositories.Snapshot, 0)
if repo.Spec.Backend.Local != nil {
snapshots, err = r.getSnapshotsFromSidecar(repo, []string{snapshotId})
} else {
Expand All @@ -77,30 +71,29 @@ func (r *REST) Get(ctx apirequest.Context, name string, options *metav1.GetOptio
return nil, errors.New("no resource found")
}

snapshot := &api.Snapshot{}
snapshot := &repositories.Snapshot{}
snapshot = &snapshots[0]
return snapshot, nil
}

func (r *REST) NewList() runtime.Object {
return &api.SnapshotList{}
return &repositories.SnapshotList{}
}

func (r *REST) List(ctx apirequest.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {

ns, ok := apirequest.NamespaceFrom(ctx)
if !ok {
return nil, errors.New("missing namespace")
}

repositories, err := r.stashClient.StashV1alpha1().Repositories(ns).List(metav1.ListOptions{})
repos, err := r.stashClient.StashV1alpha1().Repositories(ns).List(metav1.ListOptions{})
if err != nil {
return nil, err
}

var selectedRepos []v1alpha1.Repository
var selectedRepos []stash.Repository
if options.LabelSelector != nil {
for _, r := range repositories.Items {
for _, r := range repos.Items {
repoLabels := make(map[string]string)
repoLabels = r.Labels
repoLabels["repository"] = r.Name
Expand All @@ -109,12 +102,14 @@ func (r *REST) List(ctx apirequest.Context, options *metainternalversion.ListOpt
}
}
} else {
selectedRepos = repositories.Items
selectedRepos = repos.Items
}

snapshotList := &api.SnapshotList{}
snapshots := make([]api.Snapshot, 0)
snapshotList := &repositories.SnapshotList{
Items: make([]repositories.Snapshot, 0),
}
for _, repo := range selectedRepos {
var snapshots []repositories.Snapshot
if repo.Spec.Backend.Local != nil {
snapshots, err = r.getSnapshotsFromSidecar(&repo, nil)
if err != nil {
Expand All @@ -127,8 +122,10 @@ func (r *REST) List(ctx apirequest.Context, options *metainternalversion.ListOpt
}
}
snapshotList.Items = append(snapshotList.Items, snapshots...)

}

// k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructured_list.go
// unstructured.UnstructuredList{}
return snapshotList, nil
}

Expand Down
24 changes: 12 additions & 12 deletions pkg/registry/snapshot/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import (
"encoding/json"
"fmt"

api "github.com/appscode/stash/apis/repositories/v1alpha1"
"github.com/appscode/stash/apis/stash/v1alpha1"
"github.com/appscode/stash/apis/repositories"
stash "github.com/appscode/stash/apis/stash/v1alpha1"
"github.com/appscode/stash/pkg/cli"
"github.com/appscode/stash/pkg/util"
core "k8s.io/api/core/v1"
Expand All @@ -20,15 +20,15 @@ const (
ExecStash = "/bin/stash"
)

func (r *REST) GetSnapshots(repository *v1alpha1.Repository, snapshotIDs []string) ([]api.Snapshot, error) {
func (r *REST) GetSnapshots(repository *stash.Repository, snapshotIDs []string) ([]repositories.Snapshot, error) {
backend := repository.Spec.Backend.DeepCopy()

info, err := util.ExtractDataFromRepositoryLabel(repository.Labels)
if err != nil {
return nil, err
}

workload := &v1alpha1.LocalTypedReference{
workload := &stash.LocalTypedReference{
Kind: info.WorkloadKind,
Name: info.WorkloadName,
}
Expand All @@ -51,8 +51,8 @@ func (r *REST) GetSnapshots(repository *v1alpha1.Repository, snapshotIDs []strin
return nil, err
}

snapshots := make([]api.Snapshot, 0)
snapshot := &api.Snapshot{}
snapshots := make([]repositories.Snapshot, 0)
snapshot := &repositories.Snapshot{}
for _, result := range results {
snapshot.Namespace = repository.Namespace
snapshot.Name = repository.Name + "-" + result.ID[0:util.SnapshotIDLength] // snapshotName = repositoryName-first8CharacterOfSnapshotId
Expand All @@ -75,15 +75,15 @@ func (r *REST) GetSnapshots(repository *v1alpha1.Repository, snapshotIDs []strin
return snapshots, nil
}

func (r *REST) ForgetSnapshots(repository *v1alpha1.Repository, snapshotIDs []string) error {
func (r *REST) ForgetSnapshots(repository *stash.Repository, snapshotIDs []string) error {
backend := repository.Spec.Backend.DeepCopy()

info, err := util.ExtractDataFromRepositoryLabel(repository.Labels)
if err != nil {
return err
}

workload := &v1alpha1.LocalTypedReference{
workload := &stash.LocalTypedReference{
Kind: info.WorkloadKind,
Name: info.WorkloadName,
}
Expand All @@ -109,13 +109,13 @@ func (r *REST) ForgetSnapshots(repository *v1alpha1.Repository, snapshotIDs []st
return nil
}

func (r *REST) getSnapshotsFromSidecar(repository *v1alpha1.Repository, snapshotIDs []string) ([]api.Snapshot, error) {
func (r *REST) getSnapshotsFromSidecar(repository *stash.Repository, snapshotIDs []string) ([]repositories.Snapshot, error) {
response, err := r.execOnSidecar(repository, "snapshots", snapshotIDs)
if err != nil {
return nil, err
}

snapshots := make([]api.Snapshot, 0)
snapshots := make([]repositories.Snapshot, 0)
err = json.Unmarshal(response, &snapshots)
if err != nil {
return nil, err
Expand All @@ -124,15 +124,15 @@ func (r *REST) getSnapshotsFromSidecar(repository *v1alpha1.Repository, snapshot
return snapshots, nil
}

func (r *REST) forgetSnapshotsFromSidecar(repository *v1alpha1.Repository, snapshotIDs []string) error {
func (r *REST) forgetSnapshotsFromSidecar(repository *stash.Repository, snapshotIDs []string) error {
_, err := r.execOnSidecar(repository, "forget", snapshotIDs)
if err != nil {
return err
}

return nil
}
func (r *REST) execOnSidecar(repository *v1alpha1.Repository, cmd string, snapshotIDs []string) ([]byte, error) {
func (r *REST) execOnSidecar(repository *stash.Repository, cmd string, snapshotIDs []string) ([]byte, error) {
info, err := util.ExtractDataFromRepositoryLabel(repository.Labels)
if err != nil {
return nil, err
Expand Down

0 comments on commit 0da98a7

Please sign in to comment.