diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go index cd6307fdf36..943b9313f78 100644 --- a/src/controller/artifact/controller.go +++ b/src/controller/artifact/controller.go @@ -19,7 +19,6 @@ import ( "context" stderrors "errors" "fmt" - policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" "os" "strings" "time" @@ -132,7 +131,7 @@ func NewController() Controller { artrashMgr: artifactrash.Mgr, blobMgr: blob.Mgr, labelMgr: label.Mgr, - immutableMtr: rule.NewRuleMatcher(policyindex.Get), + immutableMtr: rule.NewRuleMatcher(), regCli: registry.Cli, abstractor: NewAbstractor(), accessoryMgr: accessory.Mgr, diff --git a/src/controller/tag/controller.go b/src/controller/tag/controller.go index 454557c9116..5e20419df4b 100644 --- a/src/controller/tag/controller.go +++ b/src/controller/tag/controller.go @@ -28,7 +28,6 @@ import ( "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/immutable/match" "github.com/goharbor/harbor/src/pkg/immutable/match/rule" - policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" "github.com/goharbor/harbor/src/pkg/tag" model_tag "github.com/goharbor/harbor/src/pkg/tag/model/tag" ) @@ -64,7 +63,7 @@ func NewController() Controller { return &controller{ tagMgr: tag.Mgr, artMgr: pkg.ArtifactMgr, - immutableMtr: rule.NewRuleMatcher(policyindex.Get), + immutableMtr: rule.NewRuleMatcher(), } } diff --git a/src/pkg/clients/core/artifact.go b/src/pkg/clients/core/artifact.go deleted file mode 100644 index fbcfe899f40..00000000000 --- a/src/pkg/clients/core/artifact.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package core - -import "time" - -type Artifact struct { - Digest string `json:"digest"` - Labels []Label `json:"labels"` - Tags []Tag `json:"tags"` - PullTime time.Time `json:"pull_time"` - PushTime time.Time `json:"push_time"` -} - -type Label struct { - Name string `json:"name"` -} -type Tag struct { - Name string `json:"name"` - PullTime time.Time `json:"pull_time"` - PushTime time.Time `json:"push_time"` -} diff --git a/src/pkg/clients/core/client.go b/src/pkg/clients/core/client.go index 6082d62a87e..efd5f75e332 100644 --- a/src/pkg/clients/core/client.go +++ b/src/pkg/clients/core/client.go @@ -20,6 +20,7 @@ import ( chttp "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier" + modelsv2 "github.com/goharbor/harbor/src/controller/artifact" ) // Client defines the methods that a core client should implement @@ -31,7 +32,7 @@ type Client interface { // ArtifactClient defines the methods that an image client should implement type ArtifactClient interface { - ListAllArtifacts(project, repository string) ([]*Artifact, error) + ListAllArtifacts(project, repository string) ([]*modelsv2.Artifact, error) DeleteArtifact(project, repository, digest string) error DeleteArtifactRepository(project, repository string) error } diff --git a/src/pkg/clients/core/image.go b/src/pkg/clients/core/image.go index 76cb46760c3..51051701b53 100644 --- a/src/pkg/clients/core/image.go +++ b/src/pkg/clients/core/image.go @@ -17,13 +17,14 @@ package core import ( "fmt" + modelsv2 "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/lib/encode/repository" ) -func (c *client) ListAllArtifacts(project, repo string) ([]*Artifact, error) { +func (c *client) ListAllArtifacts(project, repo string) ([]*modelsv2.Artifact, error) { repo = repository.Encode(repo) url := c.buildURL(fmt.Sprintf("/api/v2.0/projects/%s/repositories/%s/artifacts", project, repo)) - var arts []*Artifact + var arts []*modelsv2.Artifact if err := c.httpclient.GetAndIteratePagination(url, &arts); err != nil { return nil, err } diff --git a/src/pkg/immutable/match/rule/match.go b/src/pkg/immutable/match/rule/match.go index 1a1ba97325d..feccf93e73e 100644 --- a/src/pkg/immutable/match/rule/match.go +++ b/src/pkg/immutable/match/rule/match.go @@ -23,15 +23,12 @@ import ( "github.com/goharbor/harbor/src/pkg/immutable/match" "github.com/goharbor/harbor/src/pkg/immutable/model" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" + policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" ) -// EvaluatorGetter functions that gets evaluator from index based on templateID -type EvaluatorGetter func(templateID string, parameters rule.Parameters) (rule.Evaluator, error) - // Matcher ... type Matcher struct { - rules []*model.Metadata - evaluatorGetter EvaluatorGetter + rules []*model.Metadata } // Match ... @@ -91,7 +88,7 @@ func (rm *Matcher) Match(ctx context.Context, pid int64, c iselector.Candidate) params[k] = v } - evaluator, err := rm.evaluatorGetter(r.Template, params) + evaluator, err := policyindex.Get(r.Template, params) if err != nil { return false, err } @@ -118,6 +115,6 @@ func (rm *Matcher) getImmutableRules(ctx context.Context, pid int64) error { } // NewRuleMatcher ... -func NewRuleMatcher(eg EvaluatorGetter) match.ImmutableTagMatcher { - return &Matcher{evaluatorGetter: eg} +func NewRuleMatcher() match.ImmutableTagMatcher { + return &Matcher{} } diff --git a/src/pkg/immutable/match/rule/match_test.go b/src/pkg/immutable/match/rule/match_test.go index 80e01c43ec5..52398536101 100644 --- a/src/pkg/immutable/match/rule/match_test.go +++ b/src/pkg/immutable/match/rule/match_test.go @@ -6,7 +6,6 @@ import ( "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/selector" "github.com/goharbor/harbor/src/pkg/immutable/model" - policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -149,7 +148,7 @@ func (s *MatchTestSuite) TestImmuMatch() { s.ruleIDs = append(s.ruleIDs, id) s.require.Nil(err) - match := NewRuleMatcher(policyindex.Get) + match := NewRuleMatcher() c1 := selector.Candidate{ NamespaceID: 1, diff --git a/src/pkg/retention/policy/action/index/index.go b/src/pkg/retention/policy/action/index/index.go index b518d63e589..a282e8e2e2f 100644 --- a/src/pkg/retention/policy/action/index/index.go +++ b/src/pkg/retention/policy/action/index/index.go @@ -16,9 +16,8 @@ package index import ( "github.com/goharbor/harbor/src/lib/errors" - "github.com/goharbor/harbor/src/pkg/immutable/match/rule" "github.com/goharbor/harbor/src/pkg/retention/policy/action" - policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" + "github.com/goharbor/harbor/src/pkg/retention/policy/action/performer" "sync" ) @@ -27,7 +26,7 @@ var index sync.Map func init() { // Register retain action - Register(action.Retain, action.NewRetainAction(rule.NewRuleMatcher(policyindex.Get))) + Register(action.Retain, performer.NewRetainAction) } // Register the performer with the corresponding action diff --git a/src/pkg/retention/policy/action/performer.go b/src/pkg/retention/policy/action/performer.go index 773cb6286cf..74f5d7b2125 100644 --- a/src/pkg/retention/policy/action/performer.go +++ b/src/pkg/retention/policy/action/performer.go @@ -16,12 +16,7 @@ package action import ( "context" - "github.com/goharbor/harbor/src/pkg/immutable/match" - - "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/selector" - "github.com/goharbor/harbor/src/pkg/retention/dep" ) const ( @@ -44,92 +39,3 @@ type Performer interface { // PerformerFactory is factory method for creating Performer type PerformerFactory func(params interface{}, isDryRun bool) Performer - -// retainAction make sure all the candidates will be retained and others will be cleared -type retainAction struct { - all []*selector.Candidate - // Indicate if it is a dry run - isDryRun bool - - immutableTagMatcher match.ImmutableTagMatcher -} - -// Perform the action -func (ra *retainAction) Perform(ctx context.Context, candidates []*selector.Candidate) (results []*selector.Result, err error) { - retainedShare := make(map[string]bool) - immutableShare := make(map[string]bool) - for _, c := range candidates { - retainedShare[c.Hash()] = true - } - - for _, c := range ra.all { - if _, ok := retainedShare[c.Hash()]; ok { - continue - } - if ra.isImmutable(ctx, c) { - immutableShare[c.Hash()] = true - } - } - - // start to delete - if len(ra.all) > 0 { - for _, c := range ra.all { - if _, ok := retainedShare[c.Hash()]; !ok { - result := &selector.Result{ - Target: c, - } - if _, ok = immutableShare[c.Hash()]; ok { - result.Error = &selector.ImmutableError{} - } else { - if !ra.isDryRun { - if err := dep.DefaultClient.Delete(c); err != nil { - result.Error = err - } - } - } - results = append(results, result) - } - } - } - - return -} - -func (ra *retainAction) isImmutable(ctx context.Context, c *selector.Candidate) bool { - projectID := c.NamespaceID - repo := c.Repository - _, repoName := utils.ParseRepository(repo) - matched, err := ra.immutableTagMatcher.Match(ctx, projectID, selector.Candidate{ - Repository: repoName, - Tags: c.Tags, - NamespaceID: projectID, - PulledTime: c.PulledTime, - PushedTime: c.PushedTime, - }) - if err != nil { - log.Error(err) - return false - } - return matched -} - -// NewRetainAction returns factory method for RetainAction -func NewRetainAction(m match.ImmutableTagMatcher) PerformerFactory { - return func(params interface{}, isDryRun bool) Performer { - if params != nil { - if all, ok := params.([]*selector.Candidate); ok { - ra := &retainAction{ - all: all, - isDryRun: isDryRun, - immutableTagMatcher: m, - } - return ra - } - } - return &retainAction{ - all: make([]*selector.Candidate, 0), - isDryRun: isDryRun, - immutableTagMatcher: m, - } - } -} diff --git a/src/pkg/retention/policy/action/performer/performer.go b/src/pkg/retention/policy/action/performer/performer.go new file mode 100644 index 00000000000..79c15277ffe --- /dev/null +++ b/src/pkg/retention/policy/action/performer/performer.go @@ -0,0 +1,107 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package performer + +import ( + "context" + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/selector" + rule2 "github.com/goharbor/harbor/src/pkg/immutable/match/rule" + "github.com/goharbor/harbor/src/pkg/retention/dep" + "github.com/goharbor/harbor/src/pkg/retention/policy/action" +) + +// retainAction make sure all the candidates will be retained and others will be cleared +type retainAction struct { + all []*selector.Candidate + // Indicate if it is a dry run + isDryRun bool +} + +// Perform the action +func (ra *retainAction) Perform(ctx context.Context, candidates []*selector.Candidate) (results []*selector.Result, err error) { + retainedShare := make(map[string]bool) + immutableShare := make(map[string]bool) + for _, c := range candidates { + retainedShare[c.Hash()] = true + } + + for _, c := range ra.all { + if _, ok := retainedShare[c.Hash()]; ok { + continue + } + if isImmutable(ctx, c) { + immutableShare[c.Hash()] = true + } + } + + // start to delete + if len(ra.all) > 0 { + for _, c := range ra.all { + if _, ok := retainedShare[c.Hash()]; !ok { + result := &selector.Result{ + Target: c, + } + if _, ok = immutableShare[c.Hash()]; ok { + result.Error = &selector.ImmutableError{} + } else { + if !ra.isDryRun { + if err := dep.DefaultClient.Delete(c); err != nil { + result.Error = err + } + } + } + results = append(results, result) + } + } + } + + return +} + +func isImmutable(ctx context.Context, c *selector.Candidate) bool { + projectID := c.NamespaceID + repo := c.Repository + _, repoName := utils.ParseRepository(repo) + matched, err := rule2.NewRuleMatcher().Match(ctx, projectID, selector.Candidate{ + Repository: repoName, + Tags: c.Tags, + NamespaceID: projectID, + PulledTime: c.PulledTime, + PushedTime: c.PushedTime, + }) + if err != nil { + log.Error(err) + return false + } + return matched +} + +// NewRetainAction returns performer for RetainAction +func NewRetainAction(params interface{}, isDryRun bool) action.Performer { + if params != nil { + if all, ok := params.([]*selector.Candidate); ok { + return &retainAction{ + all: all, + isDryRun: isDryRun, + } + } + } + return &retainAction{ + all: make([]*selector.Candidate, 0), + isDryRun: isDryRun, + } +} diff --git a/src/pkg/retention/policy/action/performer_test.go b/src/pkg/retention/policy/action/performer/performer_test.go similarity index 95% rename from src/pkg/retention/policy/action/performer_test.go rename to src/pkg/retention/policy/action/performer/performer_test.go index 7bf55a73083..c0502a5d2a7 100644 --- a/src/pkg/retention/policy/action/performer_test.go +++ b/src/pkg/retention/policy/action/performer/performer_test.go @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package action +package performer import ( - rule2 "github.com/goharbor/harbor/src/pkg/immutable/match/rule" - policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" "testing" "time" @@ -83,7 +81,6 @@ func (suite *TestPerformerSuite) TearDownSuite() { func (suite *TestPerformerSuite) TestPerform() { p := &retainAction{ all: suite.all, - // immutableTagMatcher: rule2.NewRuleMatcher(), } candidates := []*selector.Candidate{ @@ -151,8 +148,7 @@ func (suite *TestPerformerSuite) TestPerformImmutable() { }, } p := &retainAction{ - all: all, - immutableTagMatcher: rule2.NewRuleMatcher(policyindex.Get), + all: all, } rule := &immumodel.Metadata{ diff --git a/src/pkg/retention/policy/alg/or/processor_test.go b/src/pkg/retention/policy/alg/or/processor_test.go index c98eff9ee2b..3dd7254e670 100644 --- a/src/pkg/retention/policy/alg/or/processor_test.go +++ b/src/pkg/retention/policy/alg/or/processor_test.go @@ -16,8 +16,7 @@ package or import ( "errors" - rule2 "github.com/goharbor/harbor/src/pkg/immutable/match/rule" - policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" + "github.com/goharbor/harbor/src/pkg/retention/policy/action/performer" "testing" "time" @@ -31,7 +30,6 @@ import ( "github.com/goharbor/harbor/src/lib/selector/selectors/doublestar" "github.com/goharbor/harbor/src/lib/selector/selectors/label" "github.com/goharbor/harbor/src/pkg/retention/dep" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/alg" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/always" @@ -89,8 +87,7 @@ func (suite *ProcessorTestSuite) TearDownSuite() { // TestProcess tests process method func (suite *ProcessorTestSuite) TestProcess() { - factory := action.NewRetainAction(rule2.NewRuleMatcher(policyindex.Get)) - perf := factory(suite.all, false) + perf := performer.NewRetainAction(suite.all, false) params := make([]*alg.Parameter, 0) lastxParams := make(map[string]rule.Parameter) lastxParams[lastx.ParameterX] = 10 @@ -131,8 +128,7 @@ func (suite *ProcessorTestSuite) TestProcess() { // TestProcess2 ... func (suite *ProcessorTestSuite) TestProcess2() { - factory := action.NewRetainAction(rule2.NewRuleMatcher(policyindex.Get)) - perf := factory(suite.all, false) + perf := performer.NewRetainAction(suite.all, false) params := make([]*alg.Parameter, 0) alwaysParams := make(map[string]rule.Parameter) diff --git a/src/pkg/retention/policy/builder_test.go b/src/pkg/retention/policy/builder_test.go index c1a67fef341..cecf03c34d0 100644 --- a/src/pkg/retention/policy/builder_test.go +++ b/src/pkg/retention/policy/builder_test.go @@ -15,8 +15,7 @@ package policy import ( - rule2 "github.com/goharbor/harbor/src/pkg/immutable/match/rule" - policyindex "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" + "github.com/goharbor/harbor/src/pkg/retention/policy/action/performer" "testing" "time" @@ -90,7 +89,7 @@ func (suite *TestBuilderSuite) SetupSuite() { doublestar.NSExcludes, }, doublestar.New) index.Register(label.Kind, []string{label.With, label.Without}, label.New) - index3.Register(action.Retain, action.NewRetainAction(rule2.NewRuleMatcher(policyindex.Get))) + index3.Register(action.Retain, performer.NewRetainAction) suite.oldClient = dep.DefaultClient dep.DefaultClient = &fakeRetentionClient{} diff --git a/src/pkg/retention/policy/rule/dayspl/evaluator.go b/src/pkg/retention/policy/rule/dayspl/evaluator.go index b4d3a4c92e3..b81e28671cf 100644 --- a/src/pkg/retention/policy/rule/dayspl/evaluator.go +++ b/src/pkg/retention/policy/rule/dayspl/evaluator.go @@ -16,12 +16,12 @@ package dayspl import ( "fmt" + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "time" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/selector" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" ) diff --git a/src/pkg/retention/policy/rule/daysps/evaluator.go b/src/pkg/retention/policy/rule/daysps/evaluator.go index 755cfeeedf6..6cc5f8b1764 100644 --- a/src/pkg/retention/policy/rule/daysps/evaluator.go +++ b/src/pkg/retention/policy/rule/daysps/evaluator.go @@ -16,12 +16,12 @@ package daysps import ( "fmt" + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "time" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/selector" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" ) diff --git a/src/pkg/retention/policy/rule/index/index.go b/src/pkg/retention/policy/rule/index/index.go index 1a63527293f..6f708a1a7a9 100644 --- a/src/pkg/retention/policy/rule/index/index.go +++ b/src/pkg/retention/policy/rule/index/index.go @@ -15,10 +15,10 @@ package index import ( + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "sync" "github.com/goharbor/harbor/src/lib/errors" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/always" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/dayspl" diff --git a/src/pkg/retention/policy/rule/lastx/evaluator.go b/src/pkg/retention/policy/rule/lastx/evaluator.go index e6db28e594c..3c0a7593e77 100644 --- a/src/pkg/retention/policy/rule/lastx/evaluator.go +++ b/src/pkg/retention/policy/rule/lastx/evaluator.go @@ -15,12 +15,12 @@ package lastx import ( + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "time" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/selector" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" ) diff --git a/src/pkg/retention/policy/rule/latestk/evaluator.go b/src/pkg/retention/policy/rule/latestk/evaluator.go index 72cc87a4220..9c3dc781fa2 100644 --- a/src/pkg/retention/policy/rule/latestk/evaluator.go +++ b/src/pkg/retention/policy/rule/latestk/evaluator.go @@ -15,12 +15,12 @@ package latestk import ( + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "sort" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/selector" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" ) diff --git a/src/pkg/retention/policy/rule/latestpl/evaluator.go b/src/pkg/retention/policy/rule/latestpl/evaluator.go index c550874eda7..9c62860eb0d 100644 --- a/src/pkg/retention/policy/rule/latestpl/evaluator.go +++ b/src/pkg/retention/policy/rule/latestpl/evaluator.go @@ -16,13 +16,13 @@ package latestpl import ( "fmt" + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "math" "sort" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/selector" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" ) diff --git a/src/pkg/retention/policy/rule/latestps/evaluator.go b/src/pkg/retention/policy/rule/latestps/evaluator.go index 61682dc593a..8199b990464 100644 --- a/src/pkg/retention/policy/rule/latestps/evaluator.go +++ b/src/pkg/retention/policy/rule/latestps/evaluator.go @@ -16,13 +16,13 @@ package latestps import ( "fmt" + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "math" "sort" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/selector" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" )