Skip to content

Commit ab3de70

Browse files
fix: feature gate restore, lint, and progressing timeout
Signed-off-by: Yamunadevi Shanmugam <yshanmug@redhat.com>
1 parent 6722b7e commit ab3de70

5 files changed

Lines changed: 139 additions & 197 deletions

File tree

test/extended/apiserver/featuregate.go

Lines changed: 43 additions & 154 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
exutil "github.com/openshift/origin/test/extended/util"
1919
compat_otp "github.com/openshift/origin/test/extended/util/compat_otp"
2020
"k8s.io/apimachinery/pkg/util/wait"
21+
"k8s.io/client-go/tools/clientcmd"
2122
e2e "k8s.io/kubernetes/test/e2e/framework"
2223
)
2324

@@ -133,7 +134,6 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
133134
var (
134135
testNamespace string
135136
cleanupRequired bool
136-
originalFeatureSet string
137137
originalEnabledGates string
138138
healthyStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"}
139139
progressingStatus = map[string]string{"Progressing": "True"}
@@ -188,103 +188,49 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
188188
}
189189

190190
g.By("Restoring original feature gate configuration")
191-
var restorePatch string
192-
if originalFeatureSet == "" || strings.Contains(originalFeatureSet, "Default") {
193-
restorePatch = `{"spec":{"featureSet":"","customNoUpgrade":null}}`
194-
} else if strings.Contains(originalFeatureSet, "TechPreviewNoUpgrade") || strings.Contains(originalFeatureSet, "CustomNoUpgrade") {
195-
if originalEnabledGates == "" {
196-
restorePatch = fmt.Sprintf(`{"spec":{"customNoUpgrade":{"enabled":[]}}}`)
197-
} else {
198-
var gates []string
199-
for _, gate := range strings.Fields(originalEnabledGates) {
200-
gates = append(gates, fmt.Sprintf(`"%s"`, gate))
201-
}
202-
restorePatch = fmt.Sprintf(`{"spec":{"customNoUpgrade":{"enabled":[%s]}}}`, strings.Join(gates, ","))
203-
}
204-
}
205-
if restorePatch != "" {
206-
_, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("featuregates", "cluster", "--type=merge", "-p", restorePatch).Output()
207-
if err != nil {
208-
e2e.Logf("Warning: Failed to restore feature gate: %v", err)
209-
} else {
210-
e2e.Logf("Waiting for kube-apiserver to stabilize after restoring feature gate")
211-
if err := waitForKubeAPIServer(progressingStatus, timeoutShort, "rollout started after restore"); err == nil {
212-
waitForKubeAPIServer(healthyStatus, timeoutLong, "stable after restore")
213-
}
191+
if err := restoreFeatureGateConfig(oc, originalEnabledGates); err != nil {
192+
e2e.Logf("Warning: Failed to restore feature gate: %v", err)
193+
} else {
194+
e2e.Logf("Waiting for kube-apiserver to stabilize after restoring feature gate")
195+
if err := waitForKubeAPIServer(progressingStatus, timeoutShort, "rollout started after restore"); err == nil {
196+
waitForKubeAPIServer(healthyStatus, timeoutLong, "stable after restore")
214197
}
215198
}
216199
}()
217200

218201
g.By("Saving original feature gate configuration")
219-
originalFeatureSet, err := getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.featureSet}`)
220-
o.Expect(err).NotTo(o.HaveOccurred())
221202
originalEnabledGates, _ = getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.customNoUpgrade.enabled[*]}`)
222203

223204
g.By("Creating test secret in namespace")
224-
_, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", testNamespace, "secret", "generic", testSecretName, "--from-literal=user=Bob").Output()
205+
_, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", testNamespace, "secret", "generic", testSecretName, "--from-literal=user=Bob").Output()
225206
o.Expect(err).NotTo(o.HaveOccurred())
226207

227208
secretOutput := getResourceToBeReady(oc, asAdmin, withoutNamespace, "secret", testSecretName, "-n", testNamespace)
228209
o.Expect(secretOutput).Should(o.ContainSubstring(testSecretName))
229210

230211
g.By("Enabling AllowUnsafeMalformedObjectDeletion feature gate")
231-
currentFeatureSet, err := getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.featureSet}`)
232-
o.Expect(err).NotTo(o.HaveOccurred())
233-
234-
// Get existing enabled gates to append the new gate
235-
existingGates, err := getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.customNoUpgrade.enabled[*]}`)
236-
o.Expect(err).NotTo(o.HaveOccurred())
237-
238-
// Build list of gates including existing ones
239-
gateSet := make(map[string]bool)
240-
if existingGates != "" {
241-
for _, gate := range strings.Fields(existingGates) {
242-
gateSet[gate] = true
243-
}
244-
}
245-
// Check if the gate is already present
246-
gateAlreadyEnabled := gateSet["AllowUnsafeMalformedObjectDeletion"]
247-
gateSet["AllowUnsafeMalformedObjectDeletion"] = true
248-
249-
// Convert to JSON array
250-
var gates []string
251-
for gate := range gateSet {
252-
gates = append(gates, fmt.Sprintf(`"%s"`, gate))
253-
}
254-
255-
gatesJSON := "[" + strings.Join(gates, ",") + "]"
256-
var featureGatePatch string
257-
if strings.Contains(currentFeatureSet, "TechPreviewNoUpgrade") || strings.Contains(currentFeatureSet, "CustomNoUpgrade") {
258-
featureGatePatch = fmt.Sprintf(`{"spec":{"customNoUpgrade":{"enabled":%s}}}`, gatesJSON)
259-
} else {
260-
featureGatePatch = fmt.Sprintf(`{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":%s}}}`, gatesJSON)
261-
}
262-
263-
featureGateOutput, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("featuregates", "cluster", "--type=merge", "-p", featureGatePatch).Output()
212+
alreadyEnabled, err := enableFeatureGates(oc, []string{"AllowUnsafeMalformedObjectDeletion"})
264213
o.Expect(err).NotTo(o.HaveOccurred())
265214

266-
// Only wait for rollout if the gate was actually added (not already present) and patch didn't report "no change"
267-
if !gateAlreadyEnabled && !strings.Contains(featureGateOutput, "no change") {
215+
if !alreadyEnabled {
268216
g.By("Waiting for kube-apiserver to restart due to feature gate change")
269-
// Give the operator up to 60s to start progressing. If it doesn't start within this time,
270-
// assume the feature gate is already effectively enabled and skip the rollout wait.
271-
progressErr := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 60*time.Second, false, func(ctx context.Context) (bool, error) {
217+
progressErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 300*time.Second, false, func(ctx context.Context) (bool, error) {
272218
status := getCoStatus(oc, "kube-apiserver", progressingStatus)
273219
return reflect.DeepEqual(status, progressingStatus), nil
274220
})
275221
if progressErr != nil {
276-
e2e.Logf("kube-apiserver did not start progressing within 60s, assuming feature gate already effective")
222+
e2e.Logf("kube-apiserver did not start progressing within 300s, assuming feature gate already effective")
277223
} else {
278-
// Operator started progressing, wait for it to complete
279224
compat_otp.AssertWaitPollNoErr(waitForKubeAPIServer(healthyStatus, timeoutLong, "stable after feature gate rollout"),
280225
"kube-apiserver not stable after feature gate rollout")
281226
}
282227
} else {
283-
// Gate was already enabled or no change needed, just verify operator is healthy
284-
compat_otp.AssertWaitPollNoErr(waitForKubeAPIServer(healthyStatus, timeoutShort, "stable (no gate change needed)"),
285-
"kube-apiserver not stable")
228+
e2e.Logf("Feature gate already enabled or no change needed")
286229
}
287230

231+
compat_otp.AssertWaitPollNoErr(waitForKubeAPIServer(healthyStatus, timeoutShort, "stable before corruption"),
232+
"kube-apiserver not stable before corruption")
233+
288234
g.By("Corrupting the secret in etcd")
289235
etcdCorruptCmd := fmt.Sprintf(`etcdctl put /kubernetes.io/secrets/%s/%s "%s"`, testNamespace, testSecretName, corruptedDataMarker)
290236
etcdPods := getPodsListByLabel(oc, "openshift-etcd", "etcd=true")
@@ -344,7 +290,6 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
344290
var (
345291
healthyStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"}
346292
progressingStatus = map[string]string{"Progressing": "True"}
347-
originalFeatureSet string
348293
originalEnabledGates string
349294
)
350295

@@ -360,35 +305,16 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
360305
}
361306

362307
g.By("Saving original feature gate configuration")
363-
originalFeatureSet, err = getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.featureSet}`)
364-
o.Expect(err).NotTo(o.HaveOccurred())
365308
originalEnabledGates, _ = getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.customNoUpgrade.enabled[*]}`)
366309

367310
defer func() {
368311
g.By("Restoring original feature gate configuration")
369-
var restorePatch string
370-
if originalFeatureSet == "" || strings.Contains(originalFeatureSet, "Default") {
371-
restorePatch = `{"spec":{"featureSet":"","customNoUpgrade":null}}`
372-
} else if strings.Contains(originalFeatureSet, "TechPreviewNoUpgrade") || strings.Contains(originalFeatureSet, "CustomNoUpgrade") {
373-
if originalEnabledGates == "" {
374-
restorePatch = fmt.Sprintf(`{"spec":{"customNoUpgrade":{"enabled":[]}}}`)
375-
} else {
376-
var gates []string
377-
for _, gate := range strings.Fields(originalEnabledGates) {
378-
gates = append(gates, fmt.Sprintf(`"%s"`, gate))
379-
}
380-
restorePatch = fmt.Sprintf(`{"spec":{"customNoUpgrade":{"enabled":[%s]}}}`, strings.Join(gates, ","))
381-
}
382-
}
383-
if restorePatch != "" {
384-
_, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("featuregates", "cluster", "--type=merge", "-p", restorePatch).Output()
385-
if err != nil {
386-
e2e.Logf("Warning: Failed to restore feature gate: %v", err)
387-
} else {
388-
e2e.Logf("Waiting for kube-apiserver to stabilize after restoring feature gate")
389-
if err := waitForKubeAPIServer(progressingStatus, timeoutShort, "rollout started after restore"); err == nil {
390-
waitForKubeAPIServer(healthyStatus, timeoutLong, "stable after restore")
391-
}
312+
if err := restoreFeatureGateConfig(oc, originalEnabledGates); err != nil {
313+
e2e.Logf("Warning: Failed to restore feature gate: %v", err)
314+
} else {
315+
e2e.Logf("Waiting for kube-apiserver to stabilize after restoring feature gate")
316+
if err := waitForKubeAPIServer(progressingStatus, timeoutShort, "rollout started after restore"); err == nil {
317+
waitForKubeAPIServer(healthyStatus, timeoutLong, "stable after restore")
392318
}
393319
}
394320
}()
@@ -409,81 +335,44 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
409335
g.Skip("kubeconfig is not set, hence skipping.")
410336
}
411337

412-
g.By("Get the clustername")
413-
clusterName, clusterErr := oc.AsAdmin().WithoutNamespace().Run("config").Args("view", "-o", `jsonpath={.clusters[0].name}`).Output()
414-
o.Expect(clusterErr).NotTo(o.HaveOccurred())
415-
416338
g.By("Extract TLS credentials from kubeconfig")
417-
caCmd := fmt.Sprintf(`grep certificate-authority-data: %s | awk '{print $2}' | base64 -d > %s`, kubeconfig, certCA)
418-
_, err = exec.Command("bash", "-c", caCmd).Output()
419-
o.Expect(err).NotTo(o.HaveOccurred())
420-
421-
clientkeyCmd := fmt.Sprintf(`grep client-key-data: %s | awk '{print $2}' | base64 -d > %s`, kubeconfig, clientKey)
422-
_, err = exec.Command("bash", "-c", clientkeyCmd).Output()
423-
o.Expect(err).NotTo(o.HaveOccurred())
424-
425-
clientcrtCmd := fmt.Sprintf(`grep client-certificate-data: %s | awk '{print $2}' | base64 -d > %s`, kubeconfig, clientCert)
426-
_, err = exec.Command("bash", "-c", clientcrtCmd).Output()
427-
o.Expect(err).NotTo(o.HaveOccurred())
428-
_ = clusterName
429-
430-
g.By("Enabling CBOR feature gates")
431-
currentFeatureSet, err := getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.featureSet}`)
339+
kubecfg, err := clientcmd.LoadFromFile(kubeconfig)
432340
o.Expect(err).NotTo(o.HaveOccurred())
433341

434-
// Get existing enabled gates to append CBOR gates
435-
existingGates, err := getResource(oc, asAdmin, withoutNamespace, "featuregate/cluster", "-o", `jsonpath={.spec.customNoUpgrade.enabled[*]}`)
436-
o.Expect(err).NotTo(o.HaveOccurred())
342+
currentContext := kubecfg.Contexts[kubecfg.CurrentContext]
343+
o.Expect(currentContext).NotTo(o.BeNil(), "current context not found in kubeconfig")
437344

438-
// Build list of gates including existing ones
439-
gateSet := make(map[string]bool)
440-
if existingGates != "" {
441-
for _, gate := range strings.Fields(existingGates) {
442-
gateSet[gate] = true
443-
}
444-
}
445-
// Check if all CBOR gates are already present
446-
cborGatesAlreadyEnabled := gateSet["CBORServingAndStorage"] && gateSet["ClientsAllowCBOR"] && gateSet["ClientsPreferCBOR"]
447-
448-
// Add CBOR gates
449-
gateSet["CBORServingAndStorage"] = true
450-
gateSet["ClientsAllowCBOR"] = true
451-
gateSet["ClientsPreferCBOR"] = true
452-
453-
// Convert to sorted slice for consistent patching
454-
var gates []string
455-
for gate := range gateSet {
456-
gates = append(gates, fmt.Sprintf(`"%s"`, gate))
457-
}
345+
cluster := kubecfg.Clusters[currentContext.Cluster]
346+
o.Expect(cluster).NotTo(o.BeNil(), "cluster not found in kubeconfig")
347+
o.Expect(os.WriteFile(certCA, cluster.CertificateAuthorityData, 0600)).To(o.Succeed())
458348

459-
gatesJSON := "[" + strings.Join(gates, ",") + "]"
460-
var featureGatePatch string
461-
if strings.Contains(currentFeatureSet, "TechPreviewNoUpgrade") || strings.Contains(currentFeatureSet, "CustomNoUpgrade") {
462-
featureGatePatch = fmt.Sprintf(`{"spec":{"customNoUpgrade":{"enabled":%s}}}`, gatesJSON)
463-
} else {
464-
featureGatePatch = fmt.Sprintf(`{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":%s}}}`, gatesJSON)
465-
}
349+
authInfo := kubecfg.AuthInfos[currentContext.AuthInfo]
350+
o.Expect(authInfo).NotTo(o.BeNil(), "auth info not found in kubeconfig")
351+
o.Expect(os.WriteFile(clientKey, authInfo.ClientKeyData, 0600)).To(o.Succeed())
352+
o.Expect(os.WriteFile(clientCert, authInfo.ClientCertificateData, 0600)).To(o.Succeed())
466353

467-
output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("featuregates", "cluster", "--type=merge", "-p", featureGatePatch).Output()
354+
g.By("Enabling CBOR feature gates")
355+
alreadyEnabled, err := enableFeatureGates(oc, []string{"CBORServingAndStorage", "ClientsAllowCBOR", "ClientsPreferCBOR"})
468356
o.Expect(err).NotTo(o.HaveOccurred())
469357

470-
// Only wait for rollout if CBOR gates were actually added (not already present) and patch didn't report "no change"
471-
if !cborGatesAlreadyEnabled && !strings.Contains(output, "no change") {
472-
progressErr := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 60*time.Second, false, func(ctx context.Context) (bool, error) {
358+
if !alreadyEnabled {
359+
progressErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 300*time.Second, false, func(ctx context.Context) (bool, error) {
473360
status := getCoStatus(oc, "kube-apiserver", progressingStatus)
474361
return reflect.DeepEqual(status, progressingStatus), nil
475362
})
476363
if progressErr != nil {
477-
e2e.Logf("kube-apiserver did not start progressing within 60s, assuming CBOR feature gates already effective")
364+
e2e.Logf("kube-apiserver did not start progressing within 300s, assuming CBOR feature gates already effective")
478365
} else {
479366
compat_otp.AssertWaitPollNoErr(waitForKubeAPIServer(healthyStatus, timeoutLong, "stable after CBOR feature gate rollout"),
480367
"kube-apiserver not stable after CBOR feature gate rollout")
481368
}
482369
} else {
483-
compat_otp.AssertWaitPollNoErr(waitForKubeAPIServer(healthyStatus, timeoutShort, "stable (no CBOR gate change needed)"),
484-
"kube-apiserver not stable")
370+
e2e.Logf("CBOR feature gates already enabled or no change needed")
485371
}
486372

373+
compat_otp.AssertWaitPollNoErr(waitForKubeAPIServer(healthyStatus, timeoutShort, "stable before CBOR test"),
374+
"kube-apiserver not stable before CBOR test")
375+
487376
g.By("Verifying retrieval of existing resource in CBOR format")
488377
execCmd := fmt.Sprintf(`curl --cacert %s --key %s --cert %s -X GET -H "Accept: application/cbor" $(oc whoami --show-server)/api/v1/namespaces/openshift-etcd/services/etcd --insecure`, certCA, clientKey, clientCert)
489378
curlGETCmdOutput, err := exec.Command("bash", "-c", execCmd).Output()
@@ -533,8 +422,8 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
533422
o.Expect(err).NotTo(o.HaveOccurred())
534423

535424
errDel := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 2*time.Minute, false, func(ctx context.Context) (bool, error) {
536-
podOutput, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "default", "pod", "test-pod-nginx").Output()
537-
return !strings.Contains(podOutput, `"Running"`), nil
425+
_, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "default", "pod", "test-pod-nginx").Output()
426+
return err != nil, nil
538427
})
539428
compat_otp.AssertWaitPollNoErr(errDel, "the test pod is not deleted")
540429
})

0 commit comments

Comments
 (0)