@@ -18,6 +18,7 @@ import (
1818 exutil "github.com/openshift/origin/test/extended/util"
1919 compat_otp "github.com/openshift/origin/test/extended/util/compat_otp"
2020 "k8s.io/apimachinery/pkg/util/wait"
21+ "k8s.io/client-go/tools/clientcmd"
2122 e2e "k8s.io/kubernetes/test/e2e/framework"
2223)
2324
@@ -133,7 +134,6 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
133134 var (
134135 testNamespace string
135136 cleanupRequired bool
136- originalFeatureSet string
137137 originalEnabledGates string
138138 healthyStatus = map [string ]string {"Available" : "True" , "Progressing" : "False" , "Degraded" : "False" }
139139 progressingStatus = map [string ]string {"Progressing" : "True" }
@@ -188,103 +188,49 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
188188 }
189189
190190 g .By ("Restoring original feature gate configuration" )
191- var restorePatch string
192- if originalFeatureSet == "" || strings .Contains (originalFeatureSet , "Default" ) {
193- restorePatch = `{"spec":{"featureSet":"","customNoUpgrade":null}}`
194- } else if strings .Contains (originalFeatureSet , "TechPreviewNoUpgrade" ) || strings .Contains (originalFeatureSet , "CustomNoUpgrade" ) {
195- if originalEnabledGates == "" {
196- restorePatch = fmt .Sprintf (`{"spec":{"customNoUpgrade":{"enabled":[]}}}` )
197- } else {
198- var gates []string
199- for _ , gate := range strings .Fields (originalEnabledGates ) {
200- gates = append (gates , fmt .Sprintf (`"%s"` , gate ))
201- }
202- restorePatch = fmt .Sprintf (`{"spec":{"customNoUpgrade":{"enabled":[%s]}}}` , strings .Join (gates , "," ))
203- }
204- }
205- if restorePatch != "" {
206- _ , err := oc .AsAdmin ().WithoutNamespace ().Run ("patch" ).Args ("featuregates" , "cluster" , "--type=merge" , "-p" , restorePatch ).Output ()
207- if err != nil {
208- e2e .Logf ("Warning: Failed to restore feature gate: %v" , err )
209- } else {
210- e2e .Logf ("Waiting for kube-apiserver to stabilize after restoring feature gate" )
211- if err := waitForKubeAPIServer (progressingStatus , timeoutShort , "rollout started after restore" ); err == nil {
212- waitForKubeAPIServer (healthyStatus , timeoutLong , "stable after restore" )
213- }
191+ if err := restoreFeatureGateConfig (oc , originalEnabledGates ); err != nil {
192+ e2e .Logf ("Warning: Failed to restore feature gate: %v" , err )
193+ } else {
194+ e2e .Logf ("Waiting for kube-apiserver to stabilize after restoring feature gate" )
195+ if err := waitForKubeAPIServer (progressingStatus , timeoutShort , "rollout started after restore" ); err == nil {
196+ waitForKubeAPIServer (healthyStatus , timeoutLong , "stable after restore" )
214197 }
215198 }
216199 }()
217200
218201 g .By ("Saving original feature gate configuration" )
219- originalFeatureSet , err := getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.featureSet}` )
220- o .Expect (err ).NotTo (o .HaveOccurred ())
221202 originalEnabledGates , _ = getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.customNoUpgrade.enabled[*]}` )
222203
223204 g .By ("Creating test secret in namespace" )
224- _ , err = oc .AsAdmin ().WithoutNamespace ().Run ("create" ).Args ("-n" , testNamespace , "secret" , "generic" , testSecretName , "--from-literal=user=Bob" ).Output ()
205+ _ , err : = oc .AsAdmin ().WithoutNamespace ().Run ("create" ).Args ("-n" , testNamespace , "secret" , "generic" , testSecretName , "--from-literal=user=Bob" ).Output ()
225206 o .Expect (err ).NotTo (o .HaveOccurred ())
226207
227208 secretOutput := getResourceToBeReady (oc , asAdmin , withoutNamespace , "secret" , testSecretName , "-n" , testNamespace )
228209 o .Expect (secretOutput ).Should (o .ContainSubstring (testSecretName ))
229210
230211 g .By ("Enabling AllowUnsafeMalformedObjectDeletion feature gate" )
231- currentFeatureSet , err := getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.featureSet}` )
232- o .Expect (err ).NotTo (o .HaveOccurred ())
233-
234- // Get existing enabled gates to append the new gate
235- existingGates , err := getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.customNoUpgrade.enabled[*]}` )
236- o .Expect (err ).NotTo (o .HaveOccurred ())
237-
238- // Build list of gates including existing ones
239- gateSet := make (map [string ]bool )
240- if existingGates != "" {
241- for _ , gate := range strings .Fields (existingGates ) {
242- gateSet [gate ] = true
243- }
244- }
245- // Check if the gate is already present
246- gateAlreadyEnabled := gateSet ["AllowUnsafeMalformedObjectDeletion" ]
247- gateSet ["AllowUnsafeMalformedObjectDeletion" ] = true
248-
249- // Convert to JSON array
250- var gates []string
251- for gate := range gateSet {
252- gates = append (gates , fmt .Sprintf (`"%s"` , gate ))
253- }
254-
255- gatesJSON := "[" + strings .Join (gates , "," ) + "]"
256- var featureGatePatch string
257- if strings .Contains (currentFeatureSet , "TechPreviewNoUpgrade" ) || strings .Contains (currentFeatureSet , "CustomNoUpgrade" ) {
258- featureGatePatch = fmt .Sprintf (`{"spec":{"customNoUpgrade":{"enabled":%s}}}` , gatesJSON )
259- } else {
260- featureGatePatch = fmt .Sprintf (`{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":%s}}}` , gatesJSON )
261- }
262-
263- featureGateOutput , err := oc .AsAdmin ().WithoutNamespace ().Run ("patch" ).Args ("featuregates" , "cluster" , "--type=merge" , "-p" , featureGatePatch ).Output ()
212+ alreadyEnabled , err := enableFeatureGates (oc , []string {"AllowUnsafeMalformedObjectDeletion" })
264213 o .Expect (err ).NotTo (o .HaveOccurred ())
265214
266- // Only wait for rollout if the gate was actually added (not already present) and patch didn't report "no change"
267- if ! gateAlreadyEnabled && ! strings .Contains (featureGateOutput , "no change" ) {
215+ if ! alreadyEnabled {
268216 g .By ("Waiting for kube-apiserver to restart due to feature gate change" )
269- // Give the operator up to 60s to start progressing. If it doesn't start within this time,
270- // assume the feature gate is already effectively enabled and skip the rollout wait.
271- progressErr := wait .PollUntilContextTimeout (context .Background (), 5 * time .Second , 60 * time .Second , false , func (ctx context.Context ) (bool , error ) {
217+ progressErr := wait .PollUntilContextTimeout (context .Background (), 10 * time .Second , 300 * time .Second , false , func (ctx context.Context ) (bool , error ) {
272218 status := getCoStatus (oc , "kube-apiserver" , progressingStatus )
273219 return reflect .DeepEqual (status , progressingStatus ), nil
274220 })
275221 if progressErr != nil {
276- e2e .Logf ("kube-apiserver did not start progressing within 60s , assuming feature gate already effective" )
222+ e2e .Logf ("kube-apiserver did not start progressing within 300s , assuming feature gate already effective" )
277223 } else {
278- // Operator started progressing, wait for it to complete
279224 compat_otp .AssertWaitPollNoErr (waitForKubeAPIServer (healthyStatus , timeoutLong , "stable after feature gate rollout" ),
280225 "kube-apiserver not stable after feature gate rollout" )
281226 }
282227 } else {
283- // Gate was already enabled or no change needed, just verify operator is healthy
284- compat_otp .AssertWaitPollNoErr (waitForKubeAPIServer (healthyStatus , timeoutShort , "stable (no gate change needed)" ),
285- "kube-apiserver not stable" )
228+ e2e .Logf ("Feature gate already enabled or no change needed" )
286229 }
287230
231+ compat_otp .AssertWaitPollNoErr (waitForKubeAPIServer (healthyStatus , timeoutShort , "stable before corruption" ),
232+ "kube-apiserver not stable before corruption" )
233+
288234 g .By ("Corrupting the secret in etcd" )
289235 etcdCorruptCmd := fmt .Sprintf (`etcdctl put /kubernetes.io/secrets/%s/%s "%s"` , testNamespace , testSecretName , corruptedDataMarker )
290236 etcdPods := getPodsListByLabel (oc , "openshift-etcd" , "etcd=true" )
@@ -344,7 +290,6 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
344290 var (
345291 healthyStatus = map [string ]string {"Available" : "True" , "Progressing" : "False" , "Degraded" : "False" }
346292 progressingStatus = map [string ]string {"Progressing" : "True" }
347- originalFeatureSet string
348293 originalEnabledGates string
349294 )
350295
@@ -360,35 +305,16 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
360305 }
361306
362307 g .By ("Saving original feature gate configuration" )
363- originalFeatureSet , err = getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.featureSet}` )
364- o .Expect (err ).NotTo (o .HaveOccurred ())
365308 originalEnabledGates , _ = getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.customNoUpgrade.enabled[*]}` )
366309
367310 defer func () {
368311 g .By ("Restoring original feature gate configuration" )
369- var restorePatch string
370- if originalFeatureSet == "" || strings .Contains (originalFeatureSet , "Default" ) {
371- restorePatch = `{"spec":{"featureSet":"","customNoUpgrade":null}}`
372- } else if strings .Contains (originalFeatureSet , "TechPreviewNoUpgrade" ) || strings .Contains (originalFeatureSet , "CustomNoUpgrade" ) {
373- if originalEnabledGates == "" {
374- restorePatch = fmt .Sprintf (`{"spec":{"customNoUpgrade":{"enabled":[]}}}` )
375- } else {
376- var gates []string
377- for _ , gate := range strings .Fields (originalEnabledGates ) {
378- gates = append (gates , fmt .Sprintf (`"%s"` , gate ))
379- }
380- restorePatch = fmt .Sprintf (`{"spec":{"customNoUpgrade":{"enabled":[%s]}}}` , strings .Join (gates , "," ))
381- }
382- }
383- if restorePatch != "" {
384- _ , err := oc .AsAdmin ().WithoutNamespace ().Run ("patch" ).Args ("featuregates" , "cluster" , "--type=merge" , "-p" , restorePatch ).Output ()
385- if err != nil {
386- e2e .Logf ("Warning: Failed to restore feature gate: %v" , err )
387- } else {
388- e2e .Logf ("Waiting for kube-apiserver to stabilize after restoring feature gate" )
389- if err := waitForKubeAPIServer (progressingStatus , timeoutShort , "rollout started after restore" ); err == nil {
390- waitForKubeAPIServer (healthyStatus , timeoutLong , "stable after restore" )
391- }
312+ if err := restoreFeatureGateConfig (oc , originalEnabledGates ); err != nil {
313+ e2e .Logf ("Warning: Failed to restore feature gate: %v" , err )
314+ } else {
315+ e2e .Logf ("Waiting for kube-apiserver to stabilize after restoring feature gate" )
316+ if err := waitForKubeAPIServer (progressingStatus , timeoutShort , "rollout started after restore" ); err == nil {
317+ waitForKubeAPIServer (healthyStatus , timeoutLong , "stable after restore" )
392318 }
393319 }
394320 }()
@@ -409,81 +335,44 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
409335 g .Skip ("kubeconfig is not set, hence skipping." )
410336 }
411337
412- g .By ("Get the clustername" )
413- clusterName , clusterErr := oc .AsAdmin ().WithoutNamespace ().Run ("config" ).Args ("view" , "-o" , `jsonpath={.clusters[0].name}` ).Output ()
414- o .Expect (clusterErr ).NotTo (o .HaveOccurred ())
415-
416338 g .By ("Extract TLS credentials from kubeconfig" )
417- caCmd := fmt .Sprintf (`grep certificate-authority-data: %s | awk '{print $2}' | base64 -d > %s` , kubeconfig , certCA )
418- _ , err = exec .Command ("bash" , "-c" , caCmd ).Output ()
419- o .Expect (err ).NotTo (o .HaveOccurred ())
420-
421- clientkeyCmd := fmt .Sprintf (`grep client-key-data: %s | awk '{print $2}' | base64 -d > %s` , kubeconfig , clientKey )
422- _ , err = exec .Command ("bash" , "-c" , clientkeyCmd ).Output ()
423- o .Expect (err ).NotTo (o .HaveOccurred ())
424-
425- clientcrtCmd := fmt .Sprintf (`grep client-certificate-data: %s | awk '{print $2}' | base64 -d > %s` , kubeconfig , clientCert )
426- _ , err = exec .Command ("bash" , "-c" , clientcrtCmd ).Output ()
427- o .Expect (err ).NotTo (o .HaveOccurred ())
428- _ = clusterName
429-
430- g .By ("Enabling CBOR feature gates" )
431- currentFeatureSet , err := getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.featureSet}` )
339+ kubecfg , err := clientcmd .LoadFromFile (kubeconfig )
432340 o .Expect (err ).NotTo (o .HaveOccurred ())
433341
434- // Get existing enabled gates to append CBOR gates
435- existingGates , err := getResource (oc , asAdmin , withoutNamespace , "featuregate/cluster" , "-o" , `jsonpath={.spec.customNoUpgrade.enabled[*]}` )
436- o .Expect (err ).NotTo (o .HaveOccurred ())
342+ currentContext := kubecfg .Contexts [kubecfg .CurrentContext ]
343+ o .Expect (currentContext ).NotTo (o .BeNil (), "current context not found in kubeconfig" )
437344
438- // Build list of gates including existing ones
439- gateSet := make (map [string ]bool )
440- if existingGates != "" {
441- for _ , gate := range strings .Fields (existingGates ) {
442- gateSet [gate ] = true
443- }
444- }
445- // Check if all CBOR gates are already present
446- cborGatesAlreadyEnabled := gateSet ["CBORServingAndStorage" ] && gateSet ["ClientsAllowCBOR" ] && gateSet ["ClientsPreferCBOR" ]
447-
448- // Add CBOR gates
449- gateSet ["CBORServingAndStorage" ] = true
450- gateSet ["ClientsAllowCBOR" ] = true
451- gateSet ["ClientsPreferCBOR" ] = true
452-
453- // Convert to sorted slice for consistent patching
454- var gates []string
455- for gate := range gateSet {
456- gates = append (gates , fmt .Sprintf (`"%s"` , gate ))
457- }
345+ cluster := kubecfg .Clusters [currentContext .Cluster ]
346+ o .Expect (cluster ).NotTo (o .BeNil (), "cluster not found in kubeconfig" )
347+ o .Expect (os .WriteFile (certCA , cluster .CertificateAuthorityData , 0600 )).To (o .Succeed ())
458348
459- gatesJSON := "[" + strings .Join (gates , "," ) + "]"
460- var featureGatePatch string
461- if strings .Contains (currentFeatureSet , "TechPreviewNoUpgrade" ) || strings .Contains (currentFeatureSet , "CustomNoUpgrade" ) {
462- featureGatePatch = fmt .Sprintf (`{"spec":{"customNoUpgrade":{"enabled":%s}}}` , gatesJSON )
463- } else {
464- featureGatePatch = fmt .Sprintf (`{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":%s}}}` , gatesJSON )
465- }
349+ authInfo := kubecfg .AuthInfos [currentContext .AuthInfo ]
350+ o .Expect (authInfo ).NotTo (o .BeNil (), "auth info not found in kubeconfig" )
351+ o .Expect (os .WriteFile (clientKey , authInfo .ClientKeyData , 0600 )).To (o .Succeed ())
352+ o .Expect (os .WriteFile (clientCert , authInfo .ClientCertificateData , 0600 )).To (o .Succeed ())
466353
467- output , err := oc .AsAdmin ().WithoutNamespace ().Run ("patch" ).Args ("featuregates" , "cluster" , "--type=merge" , "-p" , featureGatePatch ).Output ()
354+ g .By ("Enabling CBOR feature gates" )
355+ alreadyEnabled , err := enableFeatureGates (oc , []string {"CBORServingAndStorage" , "ClientsAllowCBOR" , "ClientsPreferCBOR" })
468356 o .Expect (err ).NotTo (o .HaveOccurred ())
469357
470- // Only wait for rollout if CBOR gates were actually added (not already present) and patch didn't report "no change"
471- if ! cborGatesAlreadyEnabled && ! strings .Contains (output , "no change" ) {
472- progressErr := wait .PollUntilContextTimeout (context .Background (), 5 * time .Second , 60 * time .Second , false , func (ctx context.Context ) (bool , error ) {
358+ if ! alreadyEnabled {
359+ progressErr := wait .PollUntilContextTimeout (context .Background (), 10 * time .Second , 300 * time .Second , false , func (ctx context.Context ) (bool , error ) {
473360 status := getCoStatus (oc , "kube-apiserver" , progressingStatus )
474361 return reflect .DeepEqual (status , progressingStatus ), nil
475362 })
476363 if progressErr != nil {
477- e2e .Logf ("kube-apiserver did not start progressing within 60s , assuming CBOR feature gates already effective" )
364+ e2e .Logf ("kube-apiserver did not start progressing within 300s , assuming CBOR feature gates already effective" )
478365 } else {
479366 compat_otp .AssertWaitPollNoErr (waitForKubeAPIServer (healthyStatus , timeoutLong , "stable after CBOR feature gate rollout" ),
480367 "kube-apiserver not stable after CBOR feature gate rollout" )
481368 }
482369 } else {
483- compat_otp .AssertWaitPollNoErr (waitForKubeAPIServer (healthyStatus , timeoutShort , "stable (no CBOR gate change needed)" ),
484- "kube-apiserver not stable" )
370+ e2e .Logf ("CBOR feature gates already enabled or no change needed" )
485371 }
486372
373+ compat_otp .AssertWaitPollNoErr (waitForKubeAPIServer (healthyStatus , timeoutShort , "stable before CBOR test" ),
374+ "kube-apiserver not stable before CBOR test" )
375+
487376 g .By ("Verifying retrieval of existing resource in CBOR format" )
488377 execCmd := fmt .Sprintf (`curl --cacert %s --key %s --cert %s -X GET -H "Accept: application/cbor" $(oc whoami --show-server)/api/v1/namespaces/openshift-etcd/services/etcd --insecure` , certCA , clientKey , clientCert )
489378 curlGETCmdOutput , err := exec .Command ("bash" , "-c" , execCmd ).Output ()
@@ -533,8 +422,8 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Feature:FeatureGate]"
533422 o .Expect (err ).NotTo (o .HaveOccurred ())
534423
535424 errDel := wait .PollUntilContextTimeout (context .Background (), 5 * time .Second , 2 * time .Minute , false , func (ctx context.Context ) (bool , error ) {
536- podOutput , _ := oc .AsAdmin ().WithoutNamespace ().Run ("get" ).Args ("-n" , "default" , "pod" , "test-pod-nginx" ).Output ()
537- return ! strings . Contains ( podOutput , `"Running"` ) , nil
425+ _ , err := oc .AsAdmin ().WithoutNamespace ().Run ("get" ).Args ("-n" , "default" , "pod" , "test-pod-nginx" ).Output ()
426+ return err != nil , nil
538427 })
539428 compat_otp .AssertWaitPollNoErr (errDel , "the test pod is not deleted" )
540429 })
0 commit comments