Skip to content

Commit 57602a3

Browse files
oliverbaehleroliverbaehler
committed
feat: some drafting
Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>
1 parent 0a09ca8 commit 57602a3

File tree

10 files changed

+188
-28
lines changed

10 files changed

+188
-28
lines changed

‎api/v1beta2/globalresourcequota_func.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ import (
77
"fmt"
88
"sort"
99

10-
"github.com/projectcapsule/capsule/pkg/api"
1110
corev1 "k8s.io/api/core/v1"
1211
"k8s.io/apimachinery/pkg/api/resource"
12+
13+
"github.com/projectcapsule/capsule/pkg/api"
1314
)
1415

1516
func (g *GlobalResourceQuota) GetQuotaSpace(index api.Name) (corev1.ResourceList, error) {

‎charts/capsule/templates/validatingwebhookconfiguration.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ webhooks:
294294
- DELETE
295295
resources:
296296
- resourcequotas
297+
- resourcequotas/status
297298
scope: 'Namespaced'
298299
sideEffects: None
299300
namespaceSelector:

‎controllers/globalquota/resourcequotas.go

+23-15
Original file line numberDiff line numberDiff line change
@@ -162,23 +162,39 @@ func (r *Manager) syncResourceQuotas(
162162
}
163163
}
164164

165-
//nolint:nestif
165+
return SyncResourceQuotas(ctx, r.Client, quota, matchingNamespaces)
166+
}
167+
168+
// Synchronize resources quotas in all the given namespaces (routines)
169+
func SyncResourceQuotas(
170+
ctx context.Context,
171+
c client.Client,
172+
quota *capsulev1beta2.GlobalResourceQuota,
173+
namespaces []string,
174+
) (err error) {
166175
group := new(errgroup.Group)
167176

168177
// Sync resource quotas for matching namespaces
169-
for _, ns := range matchingNamespaces {
178+
for _, ns := range namespaces {
170179
namespace := ns
171180

172181
group.Go(func() error {
173-
return r.syncResourceQuota(ctx, quota, namespace)
182+
return SyncResourceQuota(ctx, c, quota, namespace)
174183
})
175184
}
176185

177186
return group.Wait()
178187
}
179188

189+
// Synchronize a single resourcequota
190+
//
180191
//nolint:nakedret
181-
func (r *Manager) syncResourceQuota(ctx context.Context, quota *capsulev1beta2.GlobalResourceQuota, namespace string) (err error) {
192+
func SyncResourceQuota(
193+
ctx context.Context,
194+
c client.Client,
195+
quota *capsulev1beta2.GlobalResourceQuota,
196+
namespace string,
197+
) (err error) {
182198
// getting ResourceQuota labels for the mutateFn
183199
var quotaLabel, typeLabel string
184200

@@ -197,14 +213,12 @@ func (r *Manager) syncResourceQuota(ctx context.Context, quota *capsulev1beta2.G
197213
}
198214

199215
// Verify if quota is present
200-
if err := r.Client.Get(ctx, types.NamespacedName{Name: target.Name, Namespace: target.Namespace}, target); err != nil && !apierrors.IsNotFound(err) {
216+
if err := c.Get(ctx, types.NamespacedName{Name: target.Name, Namespace: target.Namespace}, target); err != nil && !apierrors.IsNotFound(err) {
201217
return err
202218
}
203219

204-
var res controllerutil.OperationResult
205-
206220
err = retry.RetryOnConflict(retry.DefaultBackoff, func() (retryErr error) {
207-
res, retryErr = controllerutil.CreateOrUpdate(ctx, r.Client, target, func() (err error) {
221+
_, retryErr = controllerutil.CreateOrUpdate(ctx, c, target, func() (err error) {
208222
targetLabels := target.GetLabels()
209223
if targetLabels == nil {
210224
targetLabels = map[string]string{}
@@ -229,18 +243,12 @@ func (r *Manager) syncResourceQuota(ctx context.Context, quota *capsulev1beta2.G
229243
// It may be further reduced by the limits reconciler
230244
target.Spec.Hard = space
231245

232-
r.Log.Info("Resource Quota sync result", "space", space, "name", target.Name, "namespace", target.Namespace)
233-
234-
return controllerutil.SetControllerReference(quota, target, r.Client.Scheme())
246+
return controllerutil.SetControllerReference(quota, target, c.Scheme())
235247
})
236248

237249
return retryErr
238250
})
239251

240-
r.emitEvent(quota, target.GetNamespace(), res, fmt.Sprintf("Ensuring ResourceQuota %s", target.GetName()), err)
241-
242-
r.Log.Info("Resource Quota sync result: "+string(res), "name", target.Name, "namespace", target.Namespace)
243-
244252
if err != nil {
245253
return
246254
}

‎controllers/globalquota/utils.go

+68
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,84 @@
11
package globalquota
22

33
import (
4+
"context"
45
"fmt"
56

67
corev1 "k8s.io/api/core/v1"
78
"k8s.io/apimachinery/pkg/runtime"
9+
"sigs.k8s.io/controller-runtime/pkg/client"
810
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
911

1012
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
1113
"github.com/projectcapsule/capsule/pkg/api"
14+
capsuleutils "github.com/projectcapsule/capsule/pkg/utils"
1215
)
1316

17+
// Get all matching namespaces (just names)
18+
func GetMatchingGlobalQuotaNamespacesByName(
19+
ctx context.Context,
20+
c client.Client,
21+
quota *capsulev1beta2.GlobalResourceQuota,
22+
) (nsNames []string, err error) {
23+
namespaces, err := GetMatchingGlobalQuotaNamespaces(ctx, c, quota)
24+
if err != nil {
25+
return
26+
}
27+
28+
nsNames = make([]string, 0, len(namespaces))
29+
for _, ns := range namespaces {
30+
nsNames = append(nsNames, ns.Name)
31+
}
32+
33+
return
34+
}
35+
36+
// Get all matching namespaces
37+
func GetMatchingGlobalQuotaNamespaces(
38+
ctx context.Context,
39+
c client.Client,
40+
quota *capsulev1beta2.GlobalResourceQuota,
41+
) (namespaces []corev1.Namespace, err error) {
42+
// Collect Namespaces (Matching)
43+
namespaces = make([]corev1.Namespace, 0)
44+
seenNamespaces := make(map[string]struct{})
45+
46+
// Get Item within Resource Quota
47+
objectLabel, err := capsuleutils.GetTypeLabel(&capsulev1beta2.Tenant{})
48+
if err != nil {
49+
return
50+
}
51+
52+
for _, selector := range quota.Spec.Selectors {
53+
selected, err := selector.GetMatchingNamespaces(ctx, c)
54+
if err != nil {
55+
continue
56+
}
57+
58+
for _, ns := range selected {
59+
// Skip if namespace is being deleted
60+
if !ns.ObjectMeta.DeletionTimestamp.IsZero() {
61+
continue
62+
}
63+
64+
if _, exists := seenNamespaces[ns.Name]; exists {
65+
continue // Skip duplicates
66+
}
67+
68+
if selector.MustTenantNamespace {
69+
if _, ok := ns.Labels[objectLabel]; !ok {
70+
continue
71+
}
72+
}
73+
74+
seenNamespaces[ns.Name] = struct{}{}
75+
namespaces = append(namespaces, ns)
76+
}
77+
}
78+
79+
return
80+
}
81+
1482
// Returns for an item it's name as Kubernetes object
1583
func ItemObjectName(itemName api.Name, quota *capsulev1beta2.GlobalResourceQuota) string {
1684
// Generate a name using the tenant name and item name

‎e2e/globalresourcequota_test.go

+40
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2020
"k8s.io/apimachinery/pkg/labels"
2121
"k8s.io/apimachinery/pkg/selection"
22+
"k8s.io/apimachinery/pkg/util/intstr"
2223
"k8s.io/utils/ptr"
2324
"sigs.k8s.io/controller-runtime/pkg/client"
2425

@@ -116,6 +117,11 @@ var _ = Describe("Global ResourceQuotas", func() {
116117
corev1.ResourcePods: resource.MustParse("5"),
117118
},
118119
},
120+
"connectivity": {
121+
Hard: corev1.ResourceList{
122+
corev1.ResourceServices: resource.MustParse("2"),
123+
},
124+
},
119125
},
120126
},
121127
}
@@ -167,6 +173,40 @@ var _ = Describe("Global ResourceQuotas", func() {
167173
}
168174
})
169175

176+
By("Scheduling services simultaneously in all namespaces", func() {
177+
wg := sync.WaitGroup{} // Use WaitGroup for concurrency
178+
for _, ns := range solarNs {
179+
wg.Add(1)
180+
go func(namespace string) { // Run in parallel
181+
defer wg.Done()
182+
service := &corev1.Service{
183+
ObjectMeta: metav1.ObjectMeta{
184+
Name: "test-service",
185+
Namespace: namespace,
186+
Labels: map[string]string{
187+
"test-label": "to-delete",
188+
},
189+
},
190+
Spec: corev1.ServiceSpec{
191+
// Select pods with this label (ensure these pods exist in the namespace)
192+
Selector: map[string]string{"app": "test"},
193+
Ports: []corev1.ServicePort{
194+
{
195+
Port: 80,
196+
TargetPort: intstr.FromInt(8080),
197+
Protocol: corev1.ProtocolTCP,
198+
},
199+
},
200+
Type: corev1.ServiceTypeClusterIP,
201+
},
202+
}
203+
err := k8sClient.Create(context.TODO(), service)
204+
Expect(err).Should(Succeed(), "Failed to create Service in namespace %s", namespace)
205+
}(ns)
206+
}
207+
wg.Wait() // Ensure all services are scheduled at the same time
208+
})
209+
170210
By("Scheduling deployments simultaneously in all namespaces", func() {
171211
wg := sync.WaitGroup{} // Use WaitGroup for concurrency
172212
for _, ns := range solarNs {

‎main.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -233,8 +233,7 @@ func main() {
233233
route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler(manager.GetClient())),
234234
route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))),
235235
route.Defaults(defaults.Handler(cfg, kubeVersion)),
236-
route.QuotaMutation(globalquotahook.StatusHandler(ctrl.Log.WithName("controllers").WithName("Webhook"))),
237-
route.QuotaValidation(utils.InCapsuleGroups(cfg, globalquotahook.ValidationHandler()), globalquotahook.DeletionHandler(ctrl.Log.WithName("controllers").WithName("Webhook"))),
236+
route.QuotaValidation(globalquotahook.StatusHandler(ctrl.Log.WithName("controllers").WithName("Webhook")), utils.InCapsuleGroups(cfg, globalquotahook.ValidationHandler()), globalquotahook.DeletionHandler(ctrl.Log.WithName("controllers").WithName("Webhook"))),
238237
)
239238

240239
nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion)

‎pkg/webhook/globalquota/mutating.go ‎pkg/webhook/globalquota/calculation.go

+20-10
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ func (h *statusHandler) OnUpdate(c client.Client, decoder admission.Decoder, rec
4949
}
5050

5151
func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder admission.Decoder, recorder record.EventRecorder, req admission.Request) *admission.Response {
52+
h.log.V(3).Info("loggign request", "REQUEST", req)
53+
54+
return utils.ErroredResponse(fmt.Errorf("meowie"))
55+
5256
// Decode the incoming object
5357
quota := &corev1.ResourceQuota{}
5458
if err := decoder.Decode(req, quota); err != nil {
@@ -61,7 +65,7 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
6165
return utils.ErroredResponse(fmt.Errorf("failed to decode old ResourceQuota object: %w", err))
6266
}
6367

64-
h.log.V(5).Info("loggign request", "REQUEST", req)
68+
h.log.V(3).Info("loggign request", "REQUEST", req)
6569

6670
// Get Item within Resource Quota
6771
indexLabel := capsuleutils.GetGlobalResourceQuotaTypeLabel()
@@ -83,7 +87,7 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
8387

8488
// Skip if quota not active
8589
if !globalQuota.Spec.Active {
86-
h.log.V(5).Info("GlobalQuota is not active", "quota", globalQuota.Name)
90+
h.log.V(3).Info("GlobalQuota is not active", "quota", globalQuota.Name)
8791

8892
return nil
8993
}
@@ -117,12 +121,20 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
117121
tenantUsed = corev1.ResourceList{}
118122
}
119123

120-
h.log.V(5).Info("Available space calculated", "space", availableSpace)
124+
h.log.V(3).Info("Available space calculated", "space", availableSpace)
121125

122126
// Process each resource and enforce allocation limits
123127
for resourceName, avail := range availableSpace {
124128
rlog := h.log.WithValues("resource", resourceName)
125129

130+
rlog.V(3).Info("AVAILABLE", "avail", avail, "USED", tenantUsed[resourceName], "HARD", tenantQuota.Hard[resourceName])
131+
132+
if avail.Cmp(zero) == 0 {
133+
rlog.V(3).Info("NO SPACE AVAILABLE")
134+
quota.Status.Hard[resourceName] = oldQuota.Status.Hard[resourceName]
135+
continue
136+
}
137+
126138
// Get From the status whet's currently Used
127139
var globalUsage resource.Quantity
128140
if currentUsed, exists := tenantUsed[resourceName]; exists {
@@ -148,7 +160,7 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
148160
diff := newRequested.DeepCopy()
149161
diff.Sub(oldAllocated)
150162

151-
rlog.V(5).Info("calculate ingestion", "diff", diff, "old", oldAllocated, "new", newRequested)
163+
rlog.V(3).Info("calculate ingestion", "diff", diff, "old", oldAllocated, "new", newRequested)
152164

153165
// Compare how the newly ingested resources compare against empty resources
154166
// This is the quickest way to find out, how the status must be updated
@@ -160,7 +172,7 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
160172
continue
161173
// Resource Consumtion Increased
162174
case stat > 0:
163-
rlog.V(5).Info("increase")
175+
rlog.V(3).Info("increase")
164176
// Validate Space
165177
// Overprovisioned, allocate what's left
166178
if avail.Cmp(diff) < 0 {
@@ -173,7 +185,7 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
173185

174186
//oldAllocated.Add(avail)
175187
rlog.V(5).Info("PREVENT OVERPROVISING", "allocation", oldAllocated)
176-
quota.Status.Hard[resourceName] = oldAllocated
188+
quota.Status.Hard[resourceName] = oldQuota.Status.Hard[resourceName]
177189

178190
} else {
179191
// Adding, since requested resources have space
@@ -185,7 +197,7 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
185197
}
186198
// Resource Consumption decreased
187199
default:
188-
rlog.V(5).Info("negate")
200+
rlog.V(3).Info("negate")
189201
// SUbstract Difference from available
190202
// Negative values also combine correctly with the Add() operation
191203
globalUsage.Add(diff)
@@ -197,9 +209,7 @@ func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder
197209
}
198210
}
199211

200-
rlog.V(5).Info("calculate ingestion", "diff", diff, "usage", avail, "usage", globalUsage)
201-
202-
rlog.V(5).Info("caclulated total usage", "global", globalUsage, "requested", quota.Status.Used[resourceName])
212+
rlog.V(3).Info("caclulated total usage", "global", globalUsage, "diff", diff, "usage", avail, "hard", quota.Status.Hard[resourceName], "usage", quota.Status.Used[resourceName])
203213
tenantUsed[resourceName] = globalUsage
204214
}
205215

‎pkg/webhook/globalquota/validation.go ‎pkg/webhook/globalquota/change-guard.go

File renamed without changes.

‎tnt.yaml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: capsule.clastix.io/v1beta2
2+
kind: Tenant
3+
metadata:
4+
creationTimestamp: "2025-02-18T17:38:52Z"
5+
generation: 1
6+
labels:
7+
customer-resource-pool: dev
8+
kubernetes.io/metadata.name: solar-quota
9+
name: solar-quota
10+
resourceVersion: "28140"
11+
uid: 81c4ca40-550c-4dca-97f7-6f0ca98ad88a
12+
spec:
13+
cordoned: false
14+
ingressOptions:
15+
hostnameCollisionScope: Disabled
16+
limitRanges: {}
17+
networkPolicies: {}
18+
owners:
19+
- clusterRoles:
20+
- admin
21+
- capsule-namespace-deleter
22+
kind: User
23+
name: solar-user
24+
preventDeletion: false
25+
resourceQuotas:
26+
scope: Tenant

‎zero-quota.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: v1
2+
kind: ResourceQuota
3+
metadata:
4+
name: compute-resources
5+
spec:
6+
hard:
7+
pods: "0"

0 commit comments

Comments
 (0)