From d5befb34c29d28fddb31273c6de277d89304d36b Mon Sep 17 00:00:00 2001 From: Archana Choudhary Date: Fri, 17 Jan 2025 11:51:51 +0000 Subject: [PATCH 1/6] azurefile: kata-cc: add kata node conditional --- deploy/rbac-csi-azurefile-node.yaml | 21 +++++++++++++++++++ pkg/azurefile/azurefile.go | 32 ++++++++++++++++++++++++++++- pkg/azurefile/controllerserver.go | 4 ---- pkg/azurefile/nodeserver.go | 14 +++++-------- pkg/azurefile/nodeserver_test.go | 25 +++++++++++++++------- 5 files changed, 75 insertions(+), 21 deletions(-) diff --git a/deploy/rbac-csi-azurefile-node.yaml b/deploy/rbac-csi-azurefile-node.yaml index 61f0f0c8a4..72c87e5470 100644 --- a/deploy/rbac-csi-azurefile-node.yaml +++ b/deploy/rbac-csi-azurefile-node.yaml @@ -54,3 +54,24 @@ roleRef: name: csi-azurefile-node-katacc-role apiGroup: rbac.authorization.k8s.io --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-node-role +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-node-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azurefile-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index b1e4f91910..b10f992681 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -48,6 +48,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" mount "k8s.io/mount-utils" "k8s.io/utils/ptr" @@ -195,7 +196,6 @@ const ( FSGroupChangeNone = "None" // define tag value delimiter and default is comma tagValueDelimiterField = "tagvaluedelimiter" - enableKataCCMountField = "enablekataccmount" ) var ( @@ -453,7 +453,12 @@ func (d *Driver) Run(ctx context.Context) error { csi.RegisterControllerServer(server, d) csi.RegisterNodeServer(server, d) d.server = server + val, val2, err := getNodeInfoFromLabels(ctx, d.NodeID, d.kubeClient) + if err != nil { + klog.Warningf("failed to get node info from labels: %v", err) + } + klog.V(2).Infof("Node info from labels: %s, %s", val, val2) listener, err := csicommon.ListenEndpoint(d.endpoint) if err != nil { klog.Fatalf("failed to listen endpoint: %v", err) @@ -1282,3 +1287,28 @@ func (d *Driver) getFileShareClientForSub(subscriptionID string) (fileshareclien } return d.cloud.ComputeClientFactory.GetFileShareClientForSub(subscriptionID) } + +func getNodeInfoFromLabels(ctx context.Context, nodeID string, kubeClient clientset.Interface) (string, string, error) { + if kubeClient == nil || kubeClient.CoreV1() == nil { + return "", "", fmt.Errorf("kubeClient is nil") + } + + node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeID, metav1.GetOptions{}) + if err != nil { + return "", "", fmt.Errorf("get node(%s) failed with %v", nodeID, err) + } + + if len(node.Labels) == 0 { + return "", "", fmt.Errorf("node(%s) label is empty", nodeID) + } + return node.Labels["kubernetes.azure.com/kata-mshv-vm-isolation"], node.Labels["katacontainers.io/kata-runtime"], nil +} + +func isKataNode(ctx context.Context, nodeID string, kubeClient clientset.Interface) bool { + val, val2, err := getNodeInfoFromLabels(ctx, nodeID, kubeClient) + if err != nil { + klog.Warningf("get node(%s) confidential label failed with %v", nodeID, err) + return false + } + return val == "true" || val2 == "true" +} diff --git a/pkg/azurefile/controllerserver.go b/pkg/azurefile/controllerserver.go index a1780370bf..611a468f17 100644 --- a/pkg/azurefile/controllerserver.go +++ b/pkg/azurefile/controllerserver.go @@ -264,10 +264,6 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) accountQuota = int32(value) case tagValueDelimiterField: tagValueDelimiter = v - case enableKataCCMountField: - if _, err := strconv.ParseBool(v); err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", enableKataCCMountField, v) - } default: return nil, status.Errorf(codes.InvalidArgument, "invalid parameter %q in storage class", k) } diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index c961da8188..c42077fcac 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -43,6 +43,7 @@ import ( var getRuntimeClassForPodFunc = getRuntimeClassForPod var isConfidentialRuntimeClassFunc = isConfidentialRuntimeClass +var isKataNodeFunc = isKataNode // NodePublishVolume mount the volume from staging to target path func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { @@ -101,8 +102,8 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu } if d.enableKataCCMount { - enableKataCCMount := getValueInMap(context, enableKataCCMountField) - if strings.EqualFold(enableKataCCMount, trueValue) && context[podNameField] != "" && context[podNamespaceField] != "" { + enableKataCCMount := isKataNodeFunc(ctx, d.NodeID, d.kubeClient) + if enableKataCCMount && context[podNameField] != "" && context[podNamespaceField] != "" { runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.kubeClient, context[podNameField], context[podNamespaceField]) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) @@ -252,7 +253,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe // don't respect fsType from req.GetVolumeCapability().GetMount().GetFsType() // since it's ext4 by default on Linux var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName string - var ephemeralVol, enableKataCCMount bool + var ephemeralVol bool fileShareNameReplaceMap := map[string]string{} mountPermissions := d.mountPermissions @@ -284,11 +285,6 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe fileShareNameReplaceMap[pvcNameMetadata] = v case pvNameKey: fileShareNameReplaceMap[pvNameMetadata] = v - case enableKataCCMountField: - enableKataCCMount, err = strconv.ParseBool(v) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", enableKataCCMountField, v) - } case mountPermissionsField: if v != "" { var err error @@ -423,7 +419,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } klog.V(2).Infof("volume(%s) mount %s on %s succeeded", volumeID, source, cifsMountPath) } - + enableKataCCMount := isKataNodeFunc(ctx, d.NodeID, d.kubeClient) // If runtime OS is not windows and protocol is not nfs, save mountInfo.json if d.enableKataCCMount && enableKataCCMount { if runtime.GOOS != "windows" && protocol != nfs { diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go index 4c187d6f11..5845dd5348 100644 --- a/pkg/azurefile/nodeserver_test.go +++ b/pkg/azurefile/nodeserver_test.go @@ -121,6 +121,14 @@ func mockIsConfidentialRuntimeClass(_ context.Context, _ clientset.Interface, _ return true, nil } +func mockIsKataNodeAsTrue(_ context.Context, _ string, _ clientset.Interface) bool { + return true +} + +func mockIsKataNodeAsFalse(_ context.Context, _ string, _ clientset.Interface) bool { + return false +} + func TestNodePublishVolume(t *testing.T) { d := NewFakeDriver() d.cloud = &storage.AccountRepo{} @@ -139,6 +147,7 @@ func TestNodePublishVolume(t *testing.T) { mockDirectVolume := NewMockDirectVolume(ctrl) getRuntimeClassForPodFunc = mockGetRuntimeClassForPod isConfidentialRuntimeClassFunc = mockIsConfidentialRuntimeClass + isKataNodeFunc = mockIsKataNodeAsFalse tests := []struct { desc string @@ -262,9 +271,10 @@ func TestNodePublishVolume(t *testing.T) { TargetPath: targetTest, StagingTargetPath: sourceTest, Readonly: true, - VolumeContext: map[string]string{mountPermissionsField: "0755", podNameField: "testPod", podNamespaceField: "testNamespace", enableKataCCMountField: "true"}, + VolumeContext: map[string]string{mountPermissionsField: "0755", podNameField: "testPod", podNamespaceField: "testNamespace"}, }, setup: func() { + isKataNodeFunc = mockIsKataNodeAsTrue d.directVolume = mockDirectVolume mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(&volume.MountInfo{}, nil) mockDirectVolume.EXPECT().Add(targetTest, gomock.Any()).Return(nil) @@ -460,6 +470,7 @@ func TestNodeStageVolume(t *testing.T) { defer ctrl.Finish() mockResolver := NewMockResolver(ctrl) mockDirectVolume := NewMockDirectVolume(ctrl) + isKataNodeFunc = mockIsKataNodeAsFalse tests := []struct { desc string @@ -749,6 +760,7 @@ func TestNodeStageVolume(t *testing.T) { d.resolver = mockResolver d.directVolume = mockDirectVolume if runtime.GOOS != "windows" { + isKataNodeFunc = mockIsKataNodeAsTrue mockIPAddr := &net.IPAddr{IP: net.ParseIP("192.168.1.1")} mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(nil, nil) mockResolver.EXPECT().ResolveIPAddr("ip", "test_servername").Return(mockIPAddr, nil) @@ -758,12 +770,11 @@ func TestNodeStageVolume(t *testing.T) { req: &csi.NodeStageVolumeRequest{VolumeId: "vol_1##", StagingTargetPath: sourceTest, VolumeCapability: &stdVolCap, VolumeContext: map[string]string{ - fsTypeField: "smb", - diskNameField: "test_disk.vhd", - shareNameField: "test_sharename", - serverNameField: "test_servername", - mountPermissionsField: "0755", - enableKataCCMountField: trueValue, + fsTypeField: "smb", + diskNameField: "test_disk.vhd", + shareNameField: "test_sharename", + serverNameField: "test_servername", + mountPermissionsField: "0755", }, Secrets: secrets}, skipOnWindows: true, From e8795a5efbb8f8e889efc05735be86455b79578b Mon Sep 17 00:00:00 2001 From: Archana Choudhary Date: Thu, 13 Mar 2025 08:56:48 +0000 Subject: [PATCH 2/6] add support for kata sandbox --- pkg/azurefile/nodeserver.go | 2 +- pkg/azurefile/utils.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index c42077fcac..8500448148 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -114,7 +114,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err) } if isConfidentialRuntimeClass { - klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass %s is kata-cc", volumeID, runtimeClass) + klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass is %s", volumeID, runtimeClass) source := req.GetStagingTargetPath() if len(source) == 0 { return nil, status.Error(codes.InvalidArgument, "Staging target not provided") diff --git a/pkg/azurefile/utils.go b/pkg/azurefile/utils.go index 4bad8fb8c6..3d91322ea1 100644 --- a/pkg/azurefile/utils.go +++ b/pkg/azurefile/utils.go @@ -329,6 +329,7 @@ func isReadOnlyFromCapability(vc *csi.VolumeCapability) bool { } const confidentialRuntimeClassHandler = "kata-cc" +const kataVMIsolationRuntimeClassHandler = "kata" // check if runtimeClass is confidential func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interface, runtimeClassName string) (bool, error) { @@ -345,7 +346,8 @@ func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interf return false, err } klog.Infof("runtimeClass %s handler: %s", runtimeClassName, runtimeClass.Handler) - return runtimeClass.Handler == confidentialRuntimeClassHandler, nil + return runtimeClass.Handler == confidentialRuntimeClassHandler || + runtimeClass.Handler == kataVMIsolationRuntimeClassHandler, nil } // getBackOff returns a backoff object based on the config From bb13457255eebd3da234428cbfe12510943cf3c8 Mon Sep 17 00:00:00 2001 From: Archana Choudhary Date: Thu, 13 Mar 2025 09:21:22 +0000 Subject: [PATCH 3/6] Update examples --- deploy/example/kata-cc/statefulset.yaml | 2 +- .../storageclass-azurefile-kata-cc.yaml | 20 ------------------- 2 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml diff --git a/deploy/example/kata-cc/statefulset.yaml b/deploy/example/kata-cc/statefulset.yaml index 956d4fed87..4fe180add4 100644 --- a/deploy/example/kata-cc/statefulset.yaml +++ b/deploy/example/kata-cc/statefulset.yaml @@ -37,7 +37,7 @@ spec: - metadata: name: persistent-storage spec: - storageClassName: azurefile-csi-kata-cc + storageClassName: azurefile-csi accessModes: ["ReadWriteMany"] resources: requests: diff --git a/deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml b/deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml deleted file mode 100644 index a66859f0cb..0000000000 --- a/deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: azurefile-csi-kata-cc -provisioner: file.csi.azure.com -allowVolumeExpansion: true -parameters: - skuName: Premium_LRS # available values: Premium_LRS, Premium_ZRS, Standard_LRS, Standard_GRS, Standard_ZRS, Standard_RAGRS, Standard_RAGZRS - enableKataCCMount: "true" -reclaimPolicy: Delete -volumeBindingMode: Immediate -mountOptions: - - dir_mode=0777 - - file_mode=0777 - - mfsymlinks - - cache=strict # https://linux.die.net/man/8/mount.cifs - - nosharesock # reduce probability of reconnect race - - actimeo=30 # reduce latency for metadata-heavy workload - - nobrl # disable sending byte range lock requests to the server and for applications which have challenges with posix locks From 82b8b82708e3ebc7cda72e7a130571ce0024ea6a Mon Sep 17 00:00:00 2001 From: Archana Choudhary Date: Wed, 19 Mar 2025 13:05:08 +0000 Subject: [PATCH 4/6] refactor isKataNode function and add tests --- pkg/azurefile/azurefile.go | 18 +++--- pkg/azurefile/azurefile_test.go | 94 ++++++++++++++++++++++++++++++++ pkg/azurefile/nodeserver.go | 65 +++++++++++----------- pkg/azurefile/nodeserver_test.go | 16 ++---- 4 files changed, 139 insertions(+), 54 deletions(-) diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index b10f992681..431d911ca9 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -289,6 +289,7 @@ type Driver struct { endpoint string resolver Resolver directVolume DirectVolume + isKataNode bool } // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version & @@ -330,6 +331,7 @@ func NewDriver(options *DriverOptions) *Driver { driver.endpoint = options.Endpoint driver.resolver = new(NetResolver) driver.directVolume = new(directVolume) + driver.isKataNode = false var err error getter := func(_ context.Context, _ string) (interface{}, error) { return nil, nil } @@ -453,12 +455,8 @@ func (d *Driver) Run(ctx context.Context) error { csi.RegisterControllerServer(server, d) csi.RegisterNodeServer(server, d) d.server = server - val, val2, err := getNodeInfoFromLabels(ctx, d.NodeID, d.kubeClient) - if err != nil { - klog.Warningf("failed to get node info from labels: %v", err) - } + d.isKataNode = isKataNode(ctx, d.NodeID, d.kubeClient) - klog.V(2).Infof("Node info from labels: %s, %s", val, val2) listener, err := csicommon.ListenEndpoint(d.endpoint) if err != nil { klog.Fatalf("failed to listen endpoint: %v", err) @@ -1305,10 +1303,14 @@ func getNodeInfoFromLabels(ctx context.Context, nodeID string, kubeClient client } func isKataNode(ctx context.Context, nodeID string, kubeClient clientset.Interface) bool { - val, val2, err := getNodeInfoFromLabels(ctx, nodeID, kubeClient) + kataVMIsolationLabel, kataRuntimeLabel, err := getNodeInfoFromLabels(ctx, nodeID, kubeClient) + if err != nil { - klog.Warningf("get node(%s) confidential label failed with %v", nodeID, err) + klog.Warningf("failed to get node info from labels: %v", err) return false } - return val == "true" || val2 == "true" + + klog.V(4).Infof("node(%s) labels: kataVMIsolationLabel(%s), kataRuntimeLabel(%s)", nodeID, kataVMIsolationLabel, kataRuntimeLabel) + + return strings.EqualFold(kataVMIsolationLabel, "true") || strings.EqualFold(kataRuntimeLabel, "true") } diff --git a/pkg/azurefile/azurefile_test.go b/pkg/azurefile/azurefile_test.go index 90ee62b58e..f809062557 100644 --- a/pkg/azurefile/azurefile_test.go +++ b/pkg/azurefile/azurefile_test.go @@ -1632,3 +1632,97 @@ func TestGetFileShareClientForSub(t *testing.T) { assert.Equal(t, tc.expectedError, err) } } + +func TestGetNodeInfoFromLabels(t *testing.T) { + ctx := context.TODO() + + // Test case where kubeClient is nil + _, _, err := getNodeInfoFromLabels(ctx, "test-node", nil) + if err == nil || err.Error() != "kubeClient is nil" { + t.Fatalf("expected error 'kubeClient is nil', got %v", err) + } + + // Create a fake clientset + clientset := fake.NewSimpleClientset() + + // Test case where the node does not exist + _, _, err = getNodeInfoFromLabels(ctx, "nonexistent-node", clientset) + if err == nil { + t.Fatalf("expected an error, got nil") + } + + // Test case where node exists but has no labels + node := &v1api.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{}, + }, + } + _, err = clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + _, _, err = getNodeInfoFromLabels(ctx, "test-node", clientset) + if err == nil || err.Error() != "node(test-node) label is empty" { + t.Fatalf("expected error 'node(test-node) label is empty', got %v", err) + } + + // Test case where node has kata labels + node.Labels = map[string]string{ + "kubernetes.azure.com/kata-mshv-vm-isolation": "true", + "katacontainers.io/kata-runtime": "false", + } + _, err = clientset.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + kataVMIsolation, kataRuntime, err := getNodeInfoFromLabels(ctx, "test-node", clientset) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if kataVMIsolation != "true" || kataRuntime != "false" { + t.Fatalf("expected (true, false), got (%v, %v)", kataVMIsolation, kataRuntime) + } +} + +func TestIsKataNode(t *testing.T) { + ctx := context.TODO() + clientset := fake.NewSimpleClientset() + + // Test case where node does not exist + if isKataNode(ctx, "nonexistent-node", clientset) { + t.Fatalf("expected false, got true") + } + + // Create node without kata labels + node := &v1api.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "some-other-label": "value", + }, + }, + } + _, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if isKataNode(ctx, "test-node", clientset) { + t.Fatalf("expected false, got true") + } + + // Update node with kata labels + node.Labels["kubernetes.azure.com/kata-mshv-vm-isolation"] = "true" + _, err = clientset.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if !isKataNode(ctx, "test-node", clientset) { + t.Fatalf("expected true, got false") + } +} diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index 8500448148..6c333a9aad 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -43,7 +43,6 @@ import ( var getRuntimeClassForPodFunc = getRuntimeClassForPod var isConfidentialRuntimeClassFunc = isConfidentialRuntimeClass -var isKataNodeFunc = isKataNode // NodePublishVolume mount the volume from staging to target path func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { @@ -101,42 +100,40 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu } } - if d.enableKataCCMount { - enableKataCCMount := isKataNodeFunc(ctx, d.NodeID, d.kubeClient) - if enableKataCCMount && context[podNameField] != "" && context[podNamespaceField] != "" { - runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.kubeClient, context[podNameField], context[podNamespaceField]) + enableKataCCMount := d.isKataNode && d.enableKataCCMount + if enableKataCCMount && context[podNameField] != "" && context[podNamespaceField] != "" { + runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.kubeClient, context[podNameField], context[podNamespaceField]) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) + } + klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass) + isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.kubeClient, runtimeClass) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err) + } + if isConfidentialRuntimeClass { + klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass is %s", volumeID, runtimeClass) + source := req.GetStagingTargetPath() + if len(source) == 0 { + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + // Load the mount info from staging area + mountInfo, err := d.directVolume.VolumeMountInfo(source) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) + return nil, status.Errorf(codes.Internal, "failed to load mount info from %s: %v", source, err) + } + if mountInfo == nil { + return nil, status.Errorf(codes.Internal, "mount info is nil for volume %s", volumeID) } - klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass) - isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.kubeClient, runtimeClass) + data, err := json.Marshal(mountInfo) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err) + return nil, status.Errorf(codes.Internal, "failed to marshal mount info %s: %v", source, err) } - if isConfidentialRuntimeClass { - klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass is %s", volumeID, runtimeClass) - source := req.GetStagingTargetPath() - if len(source) == 0 { - return nil, status.Error(codes.InvalidArgument, "Staging target not provided") - } - // Load the mount info from staging area - mountInfo, err := d.directVolume.VolumeMountInfo(source) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to load mount info from %s: %v", source, err) - } - if mountInfo == nil { - return nil, status.Errorf(codes.Internal, "mount info is nil for volume %s", volumeID) - } - data, err := json.Marshal(mountInfo) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to marshal mount info %s: %v", source, err) - } - if err = d.directVolume.Add(target, string(data)); err != nil { - return nil, status.Errorf(codes.Internal, "failed to save mount info %s: %v", target, err) - } - klog.V(2).Infof("NodePublishVolume: direct volume mount %s at %s successfully", source, target) - return &csi.NodePublishVolumeResponse{}, nil + if err = d.directVolume.Add(target, string(data)); err != nil { + return nil, status.Errorf(codes.Internal, "failed to save mount info %s: %v", target, err) } + klog.V(2).Infof("NodePublishVolume: direct volume mount %s at %s successfully", source, target) + return &csi.NodePublishVolumeResponse{}, nil } } } @@ -419,9 +416,9 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } klog.V(2).Infof("volume(%s) mount %s on %s succeeded", volumeID, source, cifsMountPath) } - enableKataCCMount := isKataNodeFunc(ctx, d.NodeID, d.kubeClient) + enableKataCCMount := d.isKataNode && d.enableKataCCMount // If runtime OS is not windows and protocol is not nfs, save mountInfo.json - if d.enableKataCCMount && enableKataCCMount { + if enableKataCCMount { if runtime.GOOS != "windows" && protocol != nfs { // Check if mountInfo.json is already present at the targetPath isMountInfoPresent, err := d.directVolume.VolumeMountInfo(cifsMountPath) diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go index 5845dd5348..d960533ec7 100644 --- a/pkg/azurefile/nodeserver_test.go +++ b/pkg/azurefile/nodeserver_test.go @@ -121,14 +121,6 @@ func mockIsConfidentialRuntimeClass(_ context.Context, _ clientset.Interface, _ return true, nil } -func mockIsKataNodeAsTrue(_ context.Context, _ string, _ clientset.Interface) bool { - return true -} - -func mockIsKataNodeAsFalse(_ context.Context, _ string, _ clientset.Interface) bool { - return false -} - func TestNodePublishVolume(t *testing.T) { d := NewFakeDriver() d.cloud = &storage.AccountRepo{} @@ -147,7 +139,7 @@ func TestNodePublishVolume(t *testing.T) { mockDirectVolume := NewMockDirectVolume(ctrl) getRuntimeClassForPodFunc = mockGetRuntimeClassForPod isConfidentialRuntimeClassFunc = mockIsConfidentialRuntimeClass - isKataNodeFunc = mockIsKataNodeAsFalse + d.isKataNode = false tests := []struct { desc string @@ -274,7 +266,7 @@ func TestNodePublishVolume(t *testing.T) { VolumeContext: map[string]string{mountPermissionsField: "0755", podNameField: "testPod", podNamespaceField: "testNamespace"}, }, setup: func() { - isKataNodeFunc = mockIsKataNodeAsTrue + d.isKataNode = true d.directVolume = mockDirectVolume mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(&volume.MountInfo{}, nil) mockDirectVolume.EXPECT().Add(targetTest, gomock.Any()).Return(nil) @@ -470,7 +462,7 @@ func TestNodeStageVolume(t *testing.T) { defer ctrl.Finish() mockResolver := NewMockResolver(ctrl) mockDirectVolume := NewMockDirectVolume(ctrl) - isKataNodeFunc = mockIsKataNodeAsFalse + d.isKataNode = false tests := []struct { desc string @@ -760,7 +752,7 @@ func TestNodeStageVolume(t *testing.T) { d.resolver = mockResolver d.directVolume = mockDirectVolume if runtime.GOOS != "windows" { - isKataNodeFunc = mockIsKataNodeAsTrue + d.isKataNode = true mockIPAddr := &net.IPAddr{IP: net.ParseIP("192.168.1.1")} mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(nil, nil) mockResolver.EXPECT().ResolveIPAddr("ip", "test_servername").Return(mockIPAddr, nil) From 600879fb6e5a06611cbc1174a7462f2235daf95e Mon Sep 17 00:00:00 2001 From: Archana Choudhary Date: Mon, 24 Mar 2025 13:50:37 +0000 Subject: [PATCH 5/6] Restructure tests --- pkg/azurefile/azurefile.go | 4 + pkg/azurefile/azurefile_test.go | 186 +++++++++++++++++++------------- 2 files changed, 117 insertions(+), 73 deletions(-) diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index 431d911ca9..aa39be1523 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -1303,6 +1303,10 @@ func getNodeInfoFromLabels(ctx context.Context, nodeID string, kubeClient client } func isKataNode(ctx context.Context, nodeID string, kubeClient clientset.Interface) bool { + if nodeID == "" { + return false + } + kataVMIsolationLabel, kataRuntimeLabel, err := getNodeInfoFromLabels(ctx, nodeID, kubeClient) if err != nil { diff --git a/pkg/azurefile/azurefile_test.go b/pkg/azurefile/azurefile_test.go index f809062557..56527276e4 100644 --- a/pkg/azurefile/azurefile_test.go +++ b/pkg/azurefile/azurefile_test.go @@ -39,6 +39,7 @@ import ( "go.uber.org/mock/gomock" v1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/cloud-provider-azure/pkg/azclient" @@ -1634,95 +1635,134 @@ func TestGetFileShareClientForSub(t *testing.T) { } func TestGetNodeInfoFromLabels(t *testing.T) { - ctx := context.TODO() - - // Test case where kubeClient is nil - _, _, err := getNodeInfoFromLabels(ctx, "test-node", nil) - if err == nil || err.Error() != "kubeClient is nil" { - t.Fatalf("expected error 'kubeClient is nil', got %v", err) - } - - // Create a fake clientset - clientset := fake.NewSimpleClientset() - - // Test case where the node does not exist - _, _, err = getNodeInfoFromLabels(ctx, "nonexistent-node", clientset) - if err == nil { - t.Fatalf("expected an error, got nil") - } - - // Test case where node exists but has no labels - node := &v1api.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Labels: map[string]string{}, + testCases := []struct { + name string + nodeName string + labels map[string]string + setupClient bool + expectedVals [2]string + expectedErr error + }{ + { + name: "Error when kubeClient is nil", + nodeName: "test-node", + setupClient: false, + expectedErr: fmt.Errorf("kubeClient is nil"), + }, + { + name: "Error when node does not exist", + nodeName: "nonexistent-node", + setupClient: true, + expectedErr: fmt.Errorf("get node(nonexistent-node) failed with nodes \"nonexistent-node\" not found"), + }, + { + name: "Error when node has no labels", + nodeName: "test-node", + setupClient: true, + labels: map[string]string{}, // Node exists but has no labels + expectedErr: fmt.Errorf("node(test-node) label is empty"), + }, + { + name: "Success with kata labels", + nodeName: "test-node", + setupClient: true, + labels: map[string]string{ + "kubernetes.azure.com/kata-mshv-vm-isolation": "true", + "katacontainers.io/kata-runtime": "false", + }, + expectedVals: [2]string{"true", "false"}, + expectedErr: nil, }, - } - _, err = clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("expected no error, got %v", err) } - _, _, err = getNodeInfoFromLabels(ctx, "test-node", clientset) - if err == nil || err.Error() != "node(test-node) label is empty" { - t.Fatalf("expected error 'node(test-node) label is empty', got %v", err) - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.TODO() + var clientset kubernetes.Interface - // Test case where node has kata labels - node.Labels = map[string]string{ - "kubernetes.azure.com/kata-mshv-vm-isolation": "true", - "katacontainers.io/kata-runtime": "false", - } - _, err = clientset.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } + if tc.setupClient { + clientset = fake.NewSimpleClientset() + } - kataVMIsolation, kataRuntime, err := getNodeInfoFromLabels(ctx, "test-node", clientset) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } + if tc.labels != nil && tc.setupClient { + node := &v1api.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.nodeName, + Labels: tc.labels, + }, + } + _, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + assert.NoError(t, err) + } + + kataVMIsolation, kataRuntime, err := getNodeInfoFromLabels(ctx, tc.nodeName, clientset) - if kataVMIsolation != "true" || kataRuntime != "false" { - t.Fatalf("expected (true, false), got (%v, %v)", kataVMIsolation, kataRuntime) + if tc.expectedErr != nil { + assert.EqualError(t, err, tc.expectedErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.expectedVals[0], kataVMIsolation) + assert.Equal(t, tc.expectedVals[1], kataRuntime) + } + }) } } func TestIsKataNode(t *testing.T) { - ctx := context.TODO() - clientset := fake.NewSimpleClientset() - - // Test case where node does not exist - if isKataNode(ctx, "nonexistent-node", clientset) { - t.Fatalf("expected false, got true") - } - - // Create node without kata labels - node := &v1api.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Labels: map[string]string{ + testCases := []struct { + name string + nodeName string + labels map[string]string + setupClient bool + expected bool + }{ + { + name: "Node does not exist", + nodeName: "", + setupClient: true, + expected: false, + }, + { + name: "Node exists but has no kata labels", + nodeName: "test-node", + setupClient: true, + labels: map[string]string{ "some-other-label": "value", }, + expected: false, + }, + { + name: "Node has kata labels", + nodeName: "test-node", + setupClient: true, + labels: map[string]string{ + "kubernetes.azure.com/kata-mshv-vm-isolation": "true", + }, + expected: true, }, - } - _, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("expected no error, got %v", err) } - if isKataNode(ctx, "test-node", clientset) { - t.Fatalf("expected false, got true") - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.TODO() + var clientset kubernetes.Interface - // Update node with kata labels - node.Labels["kubernetes.azure.com/kata-mshv-vm-isolation"] = "true" - _, err = clientset.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } + if tc.setupClient { + clientset = fake.NewSimpleClientset() + } - if !isKataNode(ctx, "test-node", clientset) { - t.Fatalf("expected true, got false") + if tc.labels != nil && tc.setupClient { + node := &v1api.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.nodeName, + Labels: tc.labels, + }, + } + _, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + assert.NoError(t, err) + } + result := isKataNode(ctx, tc.nodeName, clientset) + assert.Equal(t, tc.expected, result) + }) } } From 35e487cebc4b4942947bf5f7c3c569ae7edacdb6 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Tue, 25 Mar 2025 03:08:05 +0000 Subject: [PATCH 6/6] fix: add missing node get rbac fix fix fix --- charts/latest/azurefile-csi-driver-v0.0.0.tgz | Bin 14099 -> 14104 bytes .../templates/rbac-csi-azurefile-node.yaml | 3 +++ deploy/rbac-csi-azurefile-node.yaml | 24 +++--------------- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/charts/latest/azurefile-csi-driver-v0.0.0.tgz b/charts/latest/azurefile-csi-driver-v0.0.0.tgz index a566067680c5039b414ac840b76a16979f54d241..6c3a210b3b331dd2d224c4282123c46130041125 100644 GIT binary patch delta 13845 zcmV+wHtNZfZkTS6O@EXuTfIEt*icilN2hH`A;~>wPP;!ZRFxz#DgreCN)p@ch4wa<8BAu<1< zw{%_Q#(g6Xj>s>-1!1(0uC_LC6s3QLSb~rn0SKRyNb2AAl(?X`Bm^{IRANGbqch1k z9sxR}68I4NfSwJ`4xgvLpM>#<&<%NBV-PD- z0L}A%YiH-x%YX9x-`U+=&;Pr4$OMm|zk$#N7H|p?>!XR!yA$GbCfHDReKy%Z*{oGX z5#kUrK_uhTKH9weQEd8xY!*aG7n69DsuA^f6o=sn3yDANqr>45lP4SmQ1#LmI%sfP!QsTvkO-rI;u-PiPR^#3b(skuVy!nvHZ2F82$tdRFMJ(|AWM$e^^gK<>i#WN$5|Z{zH#hX}Cvg~_ z8E*RM*WWgtAP<>^YBIGwH~kr=rpOVVz=kT~p+PJpo1hS1K&ab+g8&gK8Jd8^0hV~f z*G1mz5`FaL>jpwk(B+T9!%4TNge}wg8rtuN&Wi; zvSkrAg$24MVW|4kd=j7`<7mp_E;<9LSRCR|03t(^{Q@R`Av&{sSf=fuCQLA{$JdHv znWmPsg{TRNBQa)jP~tcFeMxpV_yzXM{jVBPG-5V3a{ZOQuIo2NE@W{q(BhiOlqKlt zmqw)p^`%9Gedr@K89Xr+610L2Lw0>IiR5&UeUoy{Aqt6)MIR|mrKYol(e!*8K_8tm zJ?B0~0hX$N?3f981oE2k%T(&*mr$aCVq{zJb&X;lvjwjLSaS{?FH2J8lg*1{}(9R!s5EV|q+KU?J6bYk^LFw1RLF zVhY(SEG72Gr|{31a0s&BLpFL3R}l8mEA>}?l444KnZ(N8O@4`3P?*LM3-&67I0zPV zAB17vwV)u+j?vjC2Q}1?#z>iDSt8{>Gr+TNOWOq0ul*p--Nu^%TAOv4B zZhw-0ECg-^SCjS7B~1IMqg63yA#{t&>_kttj%9&G>T5-V>%LZxtx z{NB$>qTe3UkWl!Ah4BP_hUuQ~gAjisa(ozn0F_$&55np-rLLl)`7ldZOUtQiL9H&SHH&IxS-q2$Y-0#m#vC$%dI!>o`B% zXz6~(SsbOWheH3IjK?Y9;BTBrIHo?R>GJkW_29;avHmmrQLQ6gUy!bz%ytWIwi6!( z@VybJd88G7ocw(j4~OKYk2=R^XWkgXNhjI1zhh_nKfg~WOs9jMpj4M2|><@(GwJ4m@q0p>P}FlsYrTmABH#%rIT^{?gbizv0AC#5-v)AslNWJ zF>7C-BSyh4ogv6R+PuPiGb9(A_7~T(8mN6B>!ScJ2&PUp%&Qk@2m#|*sW&Bm4=2DU zLWb!uG;w^!v%>uNP7q{B9HF6R!j%3*lzB|mK>+^8+ce#{9S zSqp&B(cy@)^yh;c@MG<1w=CvsItM<z&*ga+(dxTbn6K+U)l&XjGC#tK9WUEp04>e2`4jD0%j zf4ai_Q%Ej8*&m;}X072V5mMEyW_~0r-1|g)wUDsxEHe>jSNXE3%GML{^;(OIDAV&Fx#CEI7)X?90E_f`tCbab1RQC{wHA) z>9Eb_(ee4gSy$f3W$KuJ75{(vdaD%wd%gQ|XC42$i|1TT=2nHMNQU`_4o!5 zg=`7|lFY;k>^$vmY@9RXkHNpJAri4bAR)1g#S0XLfCV5P)BZ-XUds@99!-1^Z@zp% z-BZxzx~Yc(ef_FfF~mGiSVj7#gQKXl7$`~=BIxVa4)VNfrDq?1=PX|QnKA!Ak0FeJ zi>{0ohXTx)|Jyq|+ok!x`+9pF|G$&x%a_gnKv!ha*GsJu1Z8QwnW%2@$IwUrvHA7u z#)d9-a1&t~=oOgSV47aEHCjxU`|`y@0SpNR)JY|)s|t6Jm%Z0}`L02k=?U8X#UM^p z$>S^FoCJXWP`k!QCTpXz5WyF96`rn$tje(BU8HNmtieC#ZQZ|N;G+#_J zw#^=(yV1bCE|=99aYE$~b^a{8KZ{PW73SX-K!yI$elw-NI)CY)&gae|En<@iQ)ZA+ zd7_ciq?FmU7?N%k*+*Z#pk$U;hpLO5UlJ-LroQ>QtYPbanK#?4M$W?=$EpOOei#Rh zjjQ3&XOoCg?H|j<)vM(q%%#VMi*$ysE4|%3Pj<59@d@TNlI-tz5;l}bYcGtpQ4diZhAFaR z$T&)I7y>najD3_LFmeKru~~*`sM;&g4gC?vbOepxp%mSrI=ZGwrPkfIhQ8fr#`^E` zz{^Jl7QzHJuK!y*uV230Ew2CDTQ6U~Ua$Xm@suW8I?mRgr)PlYc^i40Fpdb^NTAwV z(Y^f9sIaY#OG1ObMMj-kzt}!#?+IaaBLPPBOWrnr6ClgbIwPQ3Jzm5l41gE(<5jQw zqm6aph^Y!ajodozp7(FKw z5E4(KKB93Lra_u39ra1R&UKwIuxHyb8zKS?BpR8}P+c)Y*7kkgZB>H#y*7n#Tl6hB&Bx0)#DVUF*>LhhX#s z%W)rd&GxdZzdmOlMdBwj!*p8yS?ZwW1!EywVCv*?h%lmXo$*Ei0#O8ty)c+4yTpmS zQJlKmbo=xyKqtQp(2%nUx*ijMZ1GeH`D-lDGsYAA-t(jmS6d_SyM|_0?aPReKy7Gp z5MnYpGQRSx{b#!H$uEPG>I1HyLQ9IDWxV%)63ns@U7WAIe&A(&_sEB#y z$=Baw4k%*8Oav?uV1JC590Tv7pDZsQKAxQ`^<#L2KZZ%YYx{V&8o+J|}sy~S@fhJ-Y(*Srl%_|bSSR#W`s&7_)(JnM% zP<>%qkb+jcJh{YH=~k?O0@b}y4KZsX7+8w%$1uVDEJ{W9P7Z&0c~!B!z!G;LaMBX*JTWvVSSfiAE zMz4UYwZ>;7N`6miYn7lX%^gq49&N`^9GWQF3q7SLc#1e^trF85J$qOReNbi!p+lCP zR~QkQn8!YwOkzrZei zf^jm8*T8OF<=|ZYJk9)|ua~!lhQ$ew%A}$+&hBMd-vusn)`_ z25YtxsiB(dZmUSM4)2s4ofmmlnE2kLO0ua8N_^ z0(9V_yqiqVpd=d?rNJHBIG5m{df{;wuTcQWQ@c%nX(1@J-_{5Uu4zwUF>k*s!?e1m zb2Tor52^!KH$--8Swq*1n;tes!>JuRTrvfe!2O)5wp`^kWS1OF^V0NGHi zfzj-%2zU+?(KRlXlHN-dmYLVCX@QINz9^9}evWK;R9jS|n9b2x7n89>*GcbTA(&6} zl3GB47~Prg-*%&)qi2CzSmr4wjOtNQeWFi)2FtNB=4o3I(`rKRgs>gNTn*Z4FA&wE zdI3Pu1oO*^4Q1F)+WEq48nZFd91k%GHRTad(2gFZB$c7qMf%i?gQm5yE&NR%p|Qtv zWvZ}BW)x~tMX6dSAq9=`6`xh( zI;np~#+=$suU_gp#WkwqpgB4{86Y2rVHdsC3#HSu*U*Y(`BddJukVSF38$iw)nvC| z00~IMQN*~K6H=QBRh43#O-8La2&Z}`9XaiTX4N2MOfRWTg;9LWif4ShZi#CUe&C$( z8B(vyB;p=op)`R2eg%Pj73jl%1vyEwZF%8nFc~6Dr-kiUnzvyLVWd_{$xs|A<1~aT z2y=odo{=teU1amf`>gzPMJfQF1fXW8c1g=IOmkjXNoWkH;=I+ego&0!V-Mva zPW za(S5L(tyfagihWIUU6A;;wrF+%Oek$#u+Y)DqJCdhVT{;gUjLrmq!DBPuRZ|qy26R z-}i_SeaqwbR)E%92{vzOB;L0wZVw4lw<-kPituvFqvMvx!mR}Pwj!L{BS*EZ1jDvG zVr_YR+6vHUtH7Q;Y^2#Tn6fo`>>;DaR)7^-2{P<{R*BgxEU`!|i|Bgf_^nl-wN@sJ zOCqs`G(IwqQ# zq_oq2`ivS(mAS6f)C|EdNlH5P<2qmA?0kj3aUkA$*1`<^QkS|=mza^i&~j_!%8|v; zPism4g67nQ$rm=i;UK+uV!q^tIb4NJHEaZb-E{eW{-;@Jl6{cCCpx^B>`3aoCCQdU zjya1*MOdxN3!1r|0|-GTw=EMXtY^lBK+pvpa1{N9Flz4RDz?}4%F*tJn`q1SR~pEP zFvk-$Fm`po4$nv4wW@s}%IX!Si*q=MN(g5!smGsj*qvmF%dOX5dz7Ux4nabzZvG^H z4(A{YdD55zqKsxYFa!?(c~M&9)(O5Gc5-nRHjG8FpxTAE)G{ttV^QZUq-#oK>opHL zGxlXV0$Nx*r=a%p9X(S$==54wobIEaT*8|ji-Y6IIKJP%V@pZQ}cES zk?f{SwW_>SMJ+3{D=#VRB#PnL33LxCJ#+J}M zHpVmvwc4npoO4KB9f)I{9q*ySow9|sI1qV*tIio&>pCbxY_g?W59K9HJu{9I9GPu# zN$`^8xKv$^kiOL*S<)SzI8v$Adeyu623WL4KUSyR!kd^N{9_DUijq^_W^tN{tccH`Q*)^0o>VTl1ZSS6w)=s$}*)+qJaT zSRScz%3C0fDA4v^;CXiA`sn=%w&b-(?gwXw$-0q9k%f9e6 zqUBnmd1z?PQK`(g6_d3+RKk9l6cwn$s231%*>Qz!(xVQ|^Ya0CAo3f|TmD9$Ma?o6fbIir- ziri)Mc0&sXnuFwuTxt$~%I%$OZy8VE!#ZV>+Jq9L^Br_ke?t|ti(|ThiF88`8qz;SBb_~2B-op@yNLMNJViI)cXCTEZH z9Z-l>%p+CFB3Yltq3Z8Mbv{p2=l4X-kad^;6Sen&Hf}r|ZJu9$k=RpIHn*08WSZgu z=h#alahd#~cOXx~ctof&DxWvRkWUtfjy00KRylpx^RM!=<@+diO(eH z|02wAl= zGrr_y+??sHyFn4H^dh|F?@lX9X3GoHY6=^jVb+)rESRNY&Pv4-B|cIAE*@jK9t_9vw#eTpfhR+(CPAwNp4ma>>?>*fBYhUxb;G<(3*%3 zI#Gi3s;7c~7ax~Xa7@XDTu7%dEa@+&JIrM$EZ`t3=j%C(!0*mV_nCi;AWzI1ZhJAs zJUIzHwa6EH65O>bBLU&yo2W)U(cuM{EE3LheDj&QFANhpQC2(uW4+a#n2 zaTGy&>%>8MQG2D%*_J#7`P(}=EW_OFP&#*uGgoeZ-BRrZon80S!=b5+XSpm1n4w60 zcZg6S0S66c5=B5I;gF85(Ot%al}kr}HQq`K5&D16j*m9qu^N(Qys%CKppV?a1gLzW z^@qu5q%#MesW;sTrep|0c5NNN#jo4H)tLiIc;N<5q7YsrfwK@H6 zBpH*&dBDkN1n#>ZkliNf{7%o6N+)VX&XJ%tSx>q|SD5~F`?p$-i~Xh6akxR-W{XaQ z>ZRv)Mv5tw_(s*_D{G^!XLX`CJ)5tTA)t(Z69_{u+52DPsp^Gf;A}cq%w?mrrYZi8 z_mBIAGbK*Nk5hI`WigdNx(kre1$GpXmQA+!nonwQW#; zVYR!>R;hZ#=0c51LJRHJv8QJVOsVv3$udfHY|UgM0!?rLS?Ziy7G>3;IxU<(^(uotG_IiE&|Ix?uL6goNAAfJ&d{nv9@WZ}$v$QBVmZ0_Bo8RHx zn{CsxH3FbrEwYg41v9ca9eQ(1bR|vNOhLH^d7huil}htc`QB*)f;I zeShOCRR-ILgAj8Bz1A0ou200weIjOsCt<$z6EN$OFL!$KWqso1A)k0zXJ)yj%q&)9 zmdedyXsj$d%bcf@zM->7>x?MoS*MNow$BuO%cqF$$>dz8&uMwGXX$f1D>${Yk~2DU zoXBak;~wz=oTZQ3EPuG>fghz=-NBj1dw)!3sRJ>~9DVsV54kLPoaG_kT)XHwhE<yv<2j>r#KW(tevjxkZBv{qCf%PeY@+pC(&I7Cj`M(^-e_1sD(zyMVAoQ2P z;xB``Uj|>lM$*sSxHXb~jig^==GU0{HD>?Z=5Azty|9=R}9E@85n%_iwKR!@VqGdnNen^*!3(>^<7wC_;K^JoK{Y z=ZArHexTQXfBE7epaJr}e%<(C{WL%L_n5n@%oRzm<>*fF zBy3wpuh;9n-r3Rr?e%)4|8Bq9et-GuhpnC6-QLdD?$*xM554V|TiaVdpx*5^-8`|7 znE%jQx~_8LzLCdFwS9;}Hk|;KrK?>cv3a#sxU0oLUB7%m-Cqo5FDDe*x&HO*M&28# zi=ILVSU@-PJL*y)zJSoqc^pNv$s(jCH=%wQ2Y@=MWOfau4(g&m5G6EF`y9_5^^Gd8 zmu?4L^N}f~zm)9ncoL#NP?>^KCo_|TAs~NFu=L08b3I$zX~}o*9i8_&>F|~#BP7QX zMivrhmJIhdNy+nRWWP<+jpaWlZ0ZRMO_@sqe#{!tTENmOhH1%b#xFyLgF{7>$Z6N0 zFAw(%P)RAffP@vyJq#_o)&x@#)o^B>TKh7bb5exRpGO8Ib$?0wE zT2KkJIc7p0fmCg;>)~9HtY&3L5Fhq@y#qfgE0*Hx9JP!BWdWQOq+gp41=lqbeH0QJ z-!%44o>J@XMjAKIV|q*nhSW)jDZEjmt4QVIMk8fA;Lp_Db_TJ z?+K6_laL}90YsCjA}A79`8Bg{_qBiP%pXG#hlGv}N0Z|sD1Y`fCmv@Z6t6bhOyWn8 z`rP4{Vi9nx<_0>@=sPEOKm$k3Gx#nINNbKKSh%y>%*kp=))0y~VVuZm2Ewv?G0!*G zk9j^PGIyPo06*3rFe>3jW+NxS_t_*m;Y=+o`4KhC+G$KBnZUj|8`G#&BnyF;6>jdw zITUVXM92h>;FF3YA%6|a?ih;`wM5^b&Ss}reb*P{B>O^HHB1o6_!JccOBe1Jl%VW1bS5-Dtmm;3`I$^X%}K2sjas9zph~ZWm(wN zM72jw&@4HM(tol=5xTO;%VR1lqBK#znzN9KhZ2W|d&L(>FCe_Iyk@tKe(}mmrm$nN z1xxJato5z!BEz)`T0K6F@C6Bp^Nf54i za0))V0&d{&2R9Mc7hS)_B#gP)>6a-yR3jk~f<+93J60@~)%#T{A-MtyLa1HM1+>vC z$rmT;gMZbkmIl(*8DA{Cpn_RdjcwFSann*K<))+_jdTjX6v@EAKW8}}wF|D~vQQMq zQbt~Ky#7tdd>pFJ`fjINv8iLgp&XwuF4H}GrHgGXx}+B|F5fIjx!je8@YS2`xl}>1 znuo3p3f$hWLP&!RZFE5vnOLbFs3BXOky*h7=YRjQHMKJ@ysNs&z(()k3PQI}A)T#% zdNw#a{Cse|BvDK5ILhpmuobIKiFN^p&YdWQ` zw~DVL;DiP4kL!0W|NqC+gU^H0gZ+b}^TU6=_n+S$zCUp7Vcv4)NCjf1-Ul503{&SB zZwKx~FFOMcI?#^8wd}EIB?OLlEHdY+a~aXfZOkt;?(JDj$xVNAv)+EK zPEf0|^r@>8&Tgit)2Yr(!@yt9&ri-ipPU~5e}8w^UsZ)Ux+S7D0@TQ18wTg+Zyta4 zit_gE`19HR&!68Pzx(|D;Fp89H0FC^WgYs@bUEY;Arso z&%f-w|L7F$KI&{$VhwIG=X!qBZ1OwX%~XE+`1b9=>F0Mp)iKeTYdescZ+99wD1Xcu zlbc=qyU(JjlZtsYg9=DLcXDyFhES7pC=xsf0pqw<_Jt_tBrfwMM7ZfgHKjHO930bd z+Ms%b2)*dC+`HupqD!*rV^8u}$W0wp^JV1{fri?gb8dl?!6+Ml;|qsqDeIt`Eh3sN zum*0huakZ-K0W5gCE#benx3^!e@!xmxc%D~)d{?sQZWx64;Ox*Q=xzjq7u^^oi1)nM?GiRs zk?;jEY&3hU5=Rt&${h!qC&Hb4pP*c^l-M7irkBgt_y`K6Oba3w9MT~>rrLZ;7hSVT zy(dHPr+%ofluwiVpQy1A=@rl(I*7(F0j`m2L-CHYIC5BXHf+@`l&}D6b_w*<=7>$I z!Er!<`fAmRNPly4Pxap3{<$i9aAPvatTpgsf9BHv8q|4zO=!Tbg{Ri}h%+CAn6V-> z>;J8to!46>{r_rrduOfx@8YpGB&I=j+`$ZR7bw06q569);IGQYGh;tUA)!jw*yBVP zCXAlJ0;h?RN&<1T!ox%^sw+54l#pHiAW=eTrKgBY`tP&C$sDy*5jy&)UnM5`1mAq5 z_zIIyjd-Dd=hT$%7T+h`S2^UP%T|x~%sigdZnTy;#x&mvqqh9ae-^2B+*feaCr9d* z&iTxDUPnnoBcm$?$G1HRWGk=dYT<)G#gj61^E-FEW)8~b^DR|8f1drMTt;=P*XE#v zUfxTmBp$$_gH;{WNM72mI*w%d#LhTd)++8X9@9O4@lgObUDi1|M|&U7|9X6Sc>ecK zpU(b%c7E_7al94|#O3IyQ5iHGp-}I?MQlN46GnFjGpJTtc3sZ)eb;%sZH}u7Tp?oD zz{MECunRYi(*b_vrz5!b4kb*G{$IT)rVc&ionSdegZ`&aSD1hL6q1WipOnf9L4Nua z={)d%;?t+v?e(Wm$a~AUUtr|c9IH)df3)T>%y}F}L>U3CkhtnYv!OptK3lC!&#a&- zXs%Bd#L(PE?PV#ilU22KbB;?{bZdY)Md0r&l#LbJv=$BKjj$ZQ`O~xg)5Bj5PCuU> zy!(81Jox$Z$=>;2T{MiuJ4u|p|M>3kNR!xqKRk7PT*4pMMygv!aUBMrZKRrd$?UA3 zoHYPOd)b|%OWfQnp&c4IDE8(Vlbi9A?`DpLM9AE&QeyIip zkfq+%mZ_dCxhwimE3NOEVs$G)bNDL}j4bx$puGkyJ~qRmPUWVQ%nWI%XbT8Py@y_Z z8TL_8Bf*_2XftE+ZNbeF& z%4F*x;PZ79a2*9)M*-JSz;zVx5k&!48-jZPF~z1}M(Nyt)})r_AO`(UbtfULhdUsx zf+Op60t==Sm^*q=&MDw1QgeLY`0MHq^xZ|IV$j#Vey78KRp|ic-KjmGWBcoW;N<-U zCm&2~@lGe*LyGe~iYVOdWF(1182TN&by9LCNdp1SPK<6O_Dugb7OSDBRm@ z!>;4Dk34RBSLsw1j}L!)cE8O4q33xc0rzgrUZQX_^@hTWNCaMGaC2lned>Nvg+Apz zBOzEvZt2Tv6=HI+P#x;K(kD-l9D@L@yad7=(t8SZSR_D0#`RUASQ5Q|uh2In!}KXP z_tch%>3eeQEtxukFviKvhw7&bkQiOrd;RU@mxHrIq}glTuZqeNom>69v4`ic&2)45 zyKe+CVg6m4jAe$LH{}p%E%a>#e$L8D(!VmvUy;<$Bk{|0bCK3uqQ6jwyX$LkvozJx z3g?#L+l$s|F)L4nCHIPdv^Y0&RqkB9mhOoL=VKFhm4rnz7Re?)D=TKXCmu0I>o%Xc z@n3Ui|Cx>d>TPY8@LyZ6U-j1bue*3kv3Qd}%p$*5f#k9o$ySQvs?uBmipvmP9>Znm ztPxx_<8*)cEjKrd-zWR!9xsVCZCL?aEG2XoSHtEURsj_bXMi1lqU2pZ1;8ShGxP0| zcf=Z=ZU+x$b;WI=!7|n#A{MNsH@>GUGnYl0waK`<#+g0DYiqW8qEe#8H8gWYif-?2 zm}Vc8naA@&!PaQ4X3p*ltyOoQY^B)TD6JX7y4oAk7J+0{ER)PLvS~ddo3);i#ja^v zEhuV%8=4kDFl{b>gkLJFiFwdVYk<*r3NXs%;Tm4_y}^qt21qL&Z^T&~)dP%v$#kR6 zhJ1&(CBt6?d_sg!JC>^a8r_+dcys`$V*g%Zi9O#pJyYS-6#5xUJQ(N!E74j#bZN%T zO|yYbi(2H1+|KlJeLK_ocBWPwt^rTq4&Z6_rAv*NjEIweD-yy80>_M44!eIh*F80D zt(+ST<Y08;m~si3B?At5t%N1^taSN^kGJs3M^NuACnSa7BEmaIwB@s#(L$v=ADG zg2!~zm>j9=byJKnhhWpljWwpqU1Q-J>dS_53tW;_-s#SQWI0e%Eg>B9yD5sc#|->N z!wg@X#f3P3sF~nPcT|e`S&Xe(pGh-gOWgw2pr&TjTGm;0;Gzay_UNx5D>byeUh-P$ zlSWQkE3)~ojmyP@*PyBI`DuawO0LTCa#vl=1p=FK|J&}%t)24yZ@ar|{MVg4CB2qj ziRBE^) z_hItEf_Si&!?O%TOr!DBEB{cYvCuW6PbInlG{O850)&O$ouq*Kt1UD*-8TsSI0MOP znuMY!uX;NL+FqVE2}1-oL?S$tz!BywrUBaRzEazGMgyU0AyO0bv0`LMXi%tEFeJxB z7^ee&5@QyJfns;07jKT_7*q8>ipYe}Q^k9Bh5IWt9<2a^XE7f^AN8_d3XnnLS*bt* zn})866kHCd*IGEsC8(`VQb@ zkihM4G@K%dFF_y}4#Af?6=i}|BLYz-kpQ`Wk+Z;M6c$3G`TI9BAe|X|Yc~tj$v5O2 z_(XJykgdX~xk)C&;{x2Zfr6#MUNTgj;i#*V~)Otshz%x+-jS=F1 zEQD@R6fHF`QlC4+2*9xtWpr>uM8!9j4w<9o8GM%pq&3GA{oV4cR(7oQm`*>8Rh@SR z>&$i4_k``7Z$WE>uhidFT(r$rfoYk)uKv7V&y9Y$Yo1!;WVRhK?+|bR+|zoJFj{xB zL8({?6XM$YbXxnHn@xE8Wy5jcrtnpNtKM8FrnOGZejm>a{|~>wzE?a1IKLNY*8R`h z75~ppZ+mO)|GA5&w2G?n?)rMmnRERNjLC4yLMZrtbZH#RG2`TSy@Yo!f7Eu(EQgT( z(uSZsxRJmq4!wx8E22mP*Tz!G-7~M>hR2~%B_6^N(Ks;CN56JDzipJ1=&zlBD7|#w z`qXWqE8rKpzzAgL1?q%E=>J|TOIQ8Fbe8&8l>8Jx2of}b1#0#~OeU?XH)jJ~bqUG1 zQqg8aTs7MgzA_P8Hn?S*;0mZ*pbKql(#~9{S-v33++JyemLiy~$1{1i!e>8dtZ|hU zDV;uR#2oecjPZcb{LFXNrEQsiI@+leB;9CdPeB^WlkB$*&+|5_+UX}k142i)p`>g+ zic~*c*s7LOQ6j$h2l&!zDbt>_f`3Mnt!^obJ{-T|yJ_Q0S1Q(`t*Ux{8Ks>FNkivv z)QsS*t}j?h?2o^p#id@muUtuKL}XjtO6ZU+t`1u&Co@`;c~+8@tRIzs+Bb#7GF~MU z-<#r0C*E3X)~KR~sxlJ`LEgRAbQ_V&S&vRL*tq;$(`=-)1YaOtsYO z)n)ms(Xwl8TG6KYIk{G-^@^&sPQ6=|%0ZCdzda*#UbVV}Ut_5n7S|(pwFup_zPf95 zwL*D+1bV8}8~3B5)Uecl4q4hc$$8~FR#Xl4SSHiI)%3~=WO=^M`Cb;_g+!)}bbtAh zL|pGX7Vv?~Dhv z#^kdr-pT6X5Q{eJiW;VWT$md;8x8|mwtOt=4Jg^z?B-C z4SN3m{NXO9%qfk3^pM6au0{746S{MeFZJqr1~duVExXWX*#EY7u<4xzPlC*_|F?R( z<@=vrZ*9L?+y8g*l=FX-;z^mYU&_DeoM6p4K$=5U^Jo89B#p;B7xuKftfz2Xzv(kW z-XlTr(!C}wW^pWm;x-(cax<&WlXE1ATIY*;gwHbizilK~aHjr$T{-{vdY%92Zk~J7 z|CuLyt@+oQfBoFkGr#_yKN2uQ|MzxYmi7Ou-Q6|*<4&G={Qq~iIMnUJ?`El3>$&x_ Xe%8((m^?!T4UirVh&CQ)3Hnv~wY`xmq*?zh4LvQ2d=7099A5d>8wNH6sAu<1< zw{TtM#(g6Xj>s>-1!1(0t~SAb-qK9XB83s@mvoY;`xM z)fJK}pdiEvXBVzMbX1po5~)ve6>fGnUQTZuWdZE#*?bB^A|%H%v{HR+b%Do|&xb?K$0Ow0Wproi~|zjbC5#)d;usH5TNT3P)&`{A<|MQ6bTN4J{n0G ziT-+y)qgHw>jCq{y2RpAtow`($S~&MMJ(|AWMx`c^gK<>i#WN$5|Z{z*4Om!Cvg~_ z8E*RMx8K*EAP<>^YCN$$H+csWQ{)JbVNDhB&|WMg8>0|kK&ab+g8&gK85)Dc0hV~p z*G1mz5`FaL>l#8&(B+T9!%1-!*43m1@L%4p>4TNge}wg8r_$N&Wi; zvSkrAg$24MVW|4kd=j7m<7mR-E;<9LSRCL`03rjE{Q@R`0XnmMSf=fuCQLA{$JdHv znWmPsxu^+>BQaufuf%Wi`-1GQ@eAyi`(HJpXvD0o<@zgqUDt1lT*%^JPm60NQR%h z={ff?3b0gvWk*cNBaqjOU#3zgM?@e2QmV=6vp5V?p9MtVix2`OQduKWz(63%Bnlt` z8X!i|hqJ@v^EISq2q(TcVO;jn=6~*NY`Zn&2yiGzSuw>skLfY}fQ3}!tpz&y(F(#z zh$&>Ru$0&zox(q3!Xe0h583cNTtV1JFV$c9Ns1|dWfCiUH~A%EL17w4EZD6S;viVe zeGrCu*Mg>6IVMshe<9>YaUX4@e~w|S74yqp?*p;Fa``Kr9M-DSmR zb^^+oE8kwQa)$`Du6tv_x=RL9O}(2H0zWQPuhId&o$8&AlHI|8P$DN*CY5?UgAjbl zxcy0gvJkixTus(PmoVw0j#kB-h0rZ7vtvEkI+g_%sjn5a&cO{4QY0^6FaTfnQRj%A z`6CG85IU{sTg?%vd3)=%cuT5gj}u9J9PS1IqvDu`llf?_=|yp>P9Ec%j}%{F5-Npj z@)k7F@%3i_Zf}Gc%kaOg2B*+1BXd74;K!L zNq>~tEOiwX&4+2iT3SwB3u<*qtyxqn%j%u1WNQP!GUkv8)LW27+|MLdI|Vx%%uDls zMoafw&f+M2JrMfuWIRp*2Y=^8!ZGzhO_w)kst4EBjP;+{k7^z1`hs-zWVTyyvz_=T zfbR}L%_FVwC}Fr5y1f>K?AjIT%i{ERX+ zuf+2gX#e1+k8fX~!=pFHFVNq2Pmd0NkKVpO2dAgUr^-T9+p!bQZYGuq^+WdeT1aSo zQ<#jlKLT;&u#c4rL{CtFVa%uisXIZHrXuONeHh?4lupL&yBBCLjMYl@hHz2(OZD~7 z#;kpTju-{IbcP`NX#EQF^^jbw+h1JEYM}Oktd9b?AecJYFt1*q0R)U=rQVc(Je&X@ z6CsR5_g=kvbvwppE6ZnDrcSME;r6PWYF#bIgml-Q=gWgd4R5*^fD) zLu&yLIy@Xwmi~Nj1AeSM?Uu!SP3OSJxyE>CjHN$1xQRGe2}|Fh%v6y|!=S>Gwdsh- zLt1)k1-kUBqIX5OjB)5C6J_mxn$Un<3)fVS1*jQ!!kMxS(pZ6Lp$oigLS6bGov}~% z`k$^a{}hsoPxi;Bu32k%N`zE(tC=4O3->+|U(O}0+lx%Z=~ZquRoQwX?#wq4w`(V2 zX5N=9dnV>J=U3C%PGiO|LxzLYde{2}0#qLE+o|*f{l;_%5Qo2`!vT_iBS5>m`v^zc zGE{G@>x_F?2r{G)pfRR+2mvCNE+WP+M5z8=;4vTx6jO0-0hYO6ThSAu0}8AC-&^BxCX5{GX{2xhzU0Y~XhibLROSKobyYHsCm#{VRY zBOSI`KRP}?IP1zAxkw#<)8hYII~%3=-_EO-8>{%=T|DP%BIknoXNSlvDQdC~t;aWj zC}a}|kYpxSVCQLfZS9;Pe+2$z4Uvci0tty_EMA}}1S|mYnD*C_^;(9=^JwCWc>U!I z>Yjow*G)YX=<8R-iXrBC!Yb0&9UMia#XwQ25J6wRc97>?D?R&vIA`(v&y@N9c?4ku zTy$kLKNMif{NLQ(-Ym`kSE~4G{@=m#<;(hiper)&>!nr+g0eJTk5#w$Bj}_5SpWKU zZA}+DxQQ?g^a@OEFikJo8ZD;Fefi>{00x8t>ZFp@RfRjq%iim~eAl4N^aSnxVh|^) zo5 zZLvs;Oz&b+?206`oU~$)h{SFRt8)`f@o$P+<&!_r zKVv2fo&5Fd+6cmNSB%!ReAc$ZN^dvMlbvjNe1dt6B>NSQ!-f)R?S;`c>LH55FhzC@ z7)L1%L!gF#v5ztYMos`SHp?&#ReJ@xp+Dl74x#Zol%hLSN7po|)VllD(6{?cS^s?= zc=^b{T$sSd^?ze~XKU+KasA)i*zRqu*8jVBN|P-eXY0??Gr;q_wLDH3M+9ypQ0=Yg zUjArQ*v8r=p+VmwqfV_~Y#+4ugfP060HgXPZ<`5!kY#9{AyBOzFJck~zzh2Es@MI| z#=3AsJRC)wT|rP(r+w5(5Mx6kNAX41XXA2EO|^hi9Oh$t8ZWlKv-7f(0DlOh=VS~* z;&IeRG!DZwNOPs5KFQa)t`i3KY&&L6M4*90LlYXRD`v>r-Y`9BBTn`M>t7V|o#W+u zl0Ynf!Z7jNUUv1@=j@|M{A6aBPRl<_9kje)ECftiojeW@Mij0y-Y7sIia@a!1`}nMIFUDs zQ5^!aNso~CvLTG`)USw5;~wm$iU1-E! z^@V9c3R>~<7m3Is9eotnf1`c3(-Rh(K7! zIi9A|R&TL@C^50NijtmUC+DI+X?HWF;Ip;EDzO)c@l^07Q$>F`c(ocCp@_2xxFp3G zQ_f5*M4k$5d#ajm_qJv~Xw<8C8jxg_OKNCpv)9P2P`|U zFeEZDk9{^C$CQ7_i9sTgi&!!))&sbLa9xn0hq*r@5_}nRSXZ{KW}RvixjPP?q=DwA z1zs!D&^+3JY#0G$LmHtnpPE)wm*mrfv-32@tp}sr`=zy9WJ91jjL-paGl<&!(aSsy zw2<0}AIWq|Dy#9un8;Mpg;v{pde5M3Lya^Gy6BLiJv@Jga1RT(^#M>lEj-n+?S>$+ z`|E~4y)TG=076?4{s?pIOWdcwWC6#zA5@`o%!u5Txwcy9Cz2Q@@5 zKnE_$yUFwnO0scL8r-psa|sTr7an)<8U>I%wcCG`7J^dyZH=Jdn)VbH^Y*(kOsji3 zSK~7KpgM4MLu9v>HFV9m>0x6uoY=9$B~w5N+`CM*vjdn2aU5PI?au!F-~Z z)B*~`=+1oqwj2E%Jqz5zGEX^SRF8`46McU&SdNr2Puq%^Rug(BgzX^aYS31Dfv6tU z3jm77m|s?GD8qKr&KG9Wn2nLoMnQtx(1#AEyoz?ARZgXODkT zWgn>BKrF1MWs>eQFF;Q@o#Gf0CtM9(fJYBxgyY|S1lXLHk(YB772 zf?MV}x`2TatXP;TDh?O>X{`a-?0%W1shhuUp|G6|&AK%2VI#tAqB&h_hXV)AOsf$H zULYb<)h#mJ?XF{l^^|cC)sr|I^>BY;y41KX8V#HY$SyiMK0k1&?Q@3k6(a$Pm=Hv5 z7XI-8KY#o9r5*@BK{7WNrbRxzG7MZoh#6KR%H{E05LoAaL4 zI2g;0MRZ6}Frj!%d?d;~ZPtzT%$S;n#0qRJ>=ss?4mb%p>%ro8d|X^pQ@bZ^*s?X;Z!uTn(P(~ zAOVRuiWpaOLTXc?s#1)z$*2_v;Z)D0Bd2}PtQv%j=_R$PFp7^^@r;kxEpZLP51cbT zMe22#MBGCxlqL|suOP6m0)2nDASX$-EiW7mCIf`&q_7=J^EQkijMPdg8HyuioQ7}( zVNNi`Gtz~wi)Znw$XKx zsKhRvmv)s+IIQ2*sV8K^p_;m6hzipI%zMT&Z;3U=dUDM@`wro4ZjodW1^%e`5UQn614 zDi82zoCz*_auG99i2H5}&;RS$v)_8&fBycrXP>(IAOG?Ezn=e@ z{PDNv&!0W}?cImB=O+ihljnc_M&t3N`RmVTzrn%pEr~pT{$GEouyW(&pqpyP$GB(a zeR`5RH#)9mvg$AMbQi7Fer4?L9DTb_%>NQ!RC1G8x0-)BJiLNNx>17PB_#5@heEC( zkaq-uTq^W&d636tLLDCy#POSjHeP_l2H*I^c$xBM1|$X2)0fF%QIj_}g_IG20;O5} zOuIipyC;Wv?3Wqtgb)WG$dy|5t}&-Nu4eN|Xxy119Wg5Ga+nd%E!~!3W>1=mLUc2s znwp@>FHG#%hPl2Ruc|MEP5k~)R==4#i=E%o-LZ*uK%((u4bb{V`3Z;+D_V@k( zr;i9`x*TBX3QAfJC0!DDbOje}0~fsmfat@5gg!O^=%P^11wow)!Z#NNYJR(r%q4&? z7l&Cc45++C=;XcN6&FP(E(43WIP!2|oZ+IV!XW-p3242gVDlD6;(e>)_K+}j%Rt> z))vR7EdhhIokbj6K&c4N&`6& z=6I|I#;y+7;rYnBR<#dAS-rw^aSkU@3E}J}_4rc`yOS(&x%Jv>kFpfTAxLP|&7Xh7 z;T(h^Pa1PTl+o-4hTs7pFG_3ND#4e-PA<;EhOsCXRJ-t&TE^vSEb5%ObWMqDz2-q@ z#=cBPKnrW<6x4pcqerZx6%KUD23lGu+1ohX)`>PXP6L6W(L@j*hjj;k8LfAFYThm( zlHHW4R+X2ksAXk(t}OFx>9Y$cHC9hP;1m|iPW0fgJu*b=(O zMwkYnRvVR+a}KGi197af<2_WkQ?{@c2O@89)j1<;T?a*oO}2FFp}d5NXU1`iBeN|o z30|@sm#WJV(zhBUOL~_28=a3pKweF;2>&w%wFKqR6!UGxoc5M}k+&|FaE5=S?WvUq zWAlEmw(bgfeLKi&BS4G5uQi2bVX)`fi!5^ovh;n&!`yAGaNbFu-u8~-E_C=TyM|@$ z6K-W^u#COHqpH6^r?^0Z)Z<(o^7P&$7j@3V2FR@FkVJ7b$79gJ^aYqpc{stbo!l}z7fyO!1( z%Oh1zc?+Zw1=`*VJj-reAH83~mb~`J{odJOvTh_&WZ@$k8tX8`GQBExl1>GkN z=>oJJU4((BY+pdg=uj+@+@L;SH9ya7!0TnVd4grPV%4rmr!dg>yEcEcXokj^<8X_% zXt9=P9vYf+R4Vgr#bj*{m9SqXMFr|G>IFnxc3feb^r%Dg{Cogjh;ray74uw$Z64qj zZ^|k4!Xfo+G8sH+qfe;RS<{c1xE%oX2=*rTR;3-&U0f~JrtE2S<+L%R95k2W9CPuy zBzM`Y-O$2;<{-HwmzsZra(gG+8^#m(uuhqzHlf7md%TXcc#Cg=Ufl^*1*8jGK5);5fBzd~mAaPCT-Bp%YEG#7l#Gle5S9 z4k*MbW|697k*rVSQ1y4BI-e)1^LwIZ$hyn_iQ4->8#f+~HqU>qNbD&pn_J64GEMP- zbL^#&xJ>@gTaYJVJS0>ZmCv3VWca|DDq2s+z@A30AD@PPojXBDa-yr?CV0kpwljV1 zV7O))BgQLf|CBEIM=WHPAr0regtc;7GW%{nOG(oHpS4T9#kr;82c@JI`g}lA;xh?4 z`HN?ZPt93{=}Ui}Tb9(5! zaP(7hmRei?YD8>KwSWKXf8LM~%q;l#zy4Qgoqzx9e|~=onSc53fBkRkgxp0mh*ge; zxu?Q#oIBf69AOv5bj(Y_fGNUZG{QEA#Vl%lzKk!^GP=}dlXWzzHoq)7N^@fx4V*3m zgbrWSRqd$3o=i<_bbSgqiGOS3EFgmc=!_ZybhZyO=`N!oH98l1b6MqNI*FFCaRH7ba(+Ki-hwW-)yGt3&Vs?l-17vSa0=d=?-i{njE_$ z97WLHI&n~**Iub}wk1zN{&r6e%P==Pl+NAa)Rli*w^VyUr`P@Ta9}FqSuRTgW++nM z9U@dnz(IqVL=jL)IHaR%beHj9<NPrtcIi+FRar5=p%P91}a}@ z{b4d1>CAy=>P>fyDH(u}U0Vlm@!RI_b>@H)Ubw;ID1;YDATBk*k~u_#>5i15R(OsL zmF<546tN^}__c1XS{{|1h^+@NB$v=f9qjQ)bXUb}wmO4gi+~374YDFIj*epzG_&#%kppTT?IM+7H z1o%uRSyU8(`YbR>MKMK$%AM^lI;Q&j7YSl)tPH3B4&2w6{_U^guSfrFyRJC@49~Ps zt02l8-}uj0J+HUn^)?(7d%eE?|LEiSUX#uqAAfG%d{nv9@WZ}$v$QBVmY~($o8RHx zn{CsxH3FbrEwYg4IWw|39eOiMbR|vNR6)51d7hogrAqTc`QB*)f;I zeShOCRR-ITgAg+Wy;c{7u1>_vd?IFvCt<$z6ELfjFL!$KWp(1^A)k0zWoEgh%q&)9 z7Rt?HXe=!|%Z#UzzM->7tBfe-S*4Bmw$BuO%cqF$$>dz6&uMwGXW?@^OE|T&lruUr zoXBak;~wz=oQ03uEPlA=fghz=-ocs2dw)!3p#w3C9DVsV54kLOoaG_kTs!YMhGm~x zSon;>V>yv<2j>qKKW(tgvjvNuBv{tDfz>I2@+pCZ&I2q3`M(&(e^E65!npmVAoLf( z;xB@_Uj$#jLekINxD}Fqg`{6$=2w{c6=we7-Tyt^MfTm^0bUF$E`nNL7^A)@B7glG z#h*VqwE61x?Z=5Azty|9XGDju?%#e$_irx+!@VeCdnx$r)jitZ>^<7wC_;K+JoKXI z=ZArHexTQXfBE7epaJr}eqH-v^)x^DkC?-Ngy8vtct#_wOBhEQ#I>$|%oRzm<>*fE zIBZ)-uh;ACY;WuT_Ikb2e>Y!lZhyV}VPpH%&ep4)o$Z$!KlC=YHnw(tK)u^-x_M$D zG5?{ra9!obeIt*VYWom{Y%&HaOIN!@V*P5Pa94|gx_Kj#F zFWnBh<|9)|e<|6ocpRcXQJI2LCsUJzAs~N_vGhmpb3I$zX~}o*9i6w6ba=~=5t3sG zLko#BONRTKq~!S|vfn1^#`2#NHuZ#srpzP(KW2?+EnsOCgS6x|#=}7>PWQ#D9~>qQV|Q@=S9>?PdAgFA7>EIhy}Z48P$xJD68R4QVFwl|T0m1`)Db^&3 z?+K7=laL}90a%l%A}A7D`8Bg{_qBiP%pXAzhlCCfhm+$XD1Y=dCmv@Z6t6a0PvS?B z`rP4{Vi9nx<_0>@=sPEOKm$k3Q}`|oNNbMASh%y>%*bj<))0y~VVuZG2Ewv?G0!*G zk9j^PGIyPo06*3rFe>3jW+NxS_t`i);Y=+o`4KhC+G$KB8Njdw z85C}1M93Hq;ggCZA%8W??g)z$wM5^b&U&XO^HHB1o6_yiRMOBe1Jl%VW%XSfnJq|${rsyLs8Of+J)FhYOAgOuPc~;Sr#@m zQSFfvG)<18w0~?-gsyDz@|ensD2>&x<}9S*p~Ru#UhxIe3kWYPui34mU%axCDePEm z!4kVUYkgz0$Z)NKR*#P(d_h9uJfy=}R9&if-@kY2tbEe7MxB)g=hJRK%RXI}@|vNN ze(TR>R%f@*MLPx8R+Zh2CUdLXyH*HI^SEKAs;7RM&wn?MbEA9z^O%Dmf3&tcPEF~h z+Z;~1&b2=&Ilry3U|q|dg3ift7xVI|XKsu4%*5~rjRkl*!2eo@Bc4`1$XO(!`nG5r zoPy7;fEzgc!A*qqMb~dI31e<{`eh0a)ksK$U>*bEjune#^?sE~NUnf_5NcO*0d4e3 z^2LezV1K!)rGa#H#up3EsbH2>V;eP7+_cn5xhbhfBb~xqA{iL?=QPKocEOcg7K-9X z%E(KO*S`*#k3;oY-|ciOHgyCzl%o^IWx8jtbg`{Pm-HgW`NurkAag^yRVJlXf7AY6sroF2jU)iKeTX*-aaZ+99wD1Xcu zlj~jlhtHyklZtsYg$hVNcXDyPhES7pC=xsX0pqw<_Jt_tBrfwMM7ZulHKjHO930bd z(x7^T2)*dC+`HupqD!*wV^8u}$aNi6^JV1{fri?gb8e24!6+MluqwLsk~x*w6fw<7;eCT+zk_Ec{I^{`XTg5| zFctsZ+vsh*D&fC3H@AB${P$fvp63-H-<2%7YX%{{cXntKbT@**i*Ae(#Culkb_tuR zNce&nHkv(Fi6aVs<&Fc*65&q1Pf)H{O6-qL)63;+d<2D3rUelT4(Wg$Q*Az_i>_Iv z-jf0N6F<~f%BM;GPt;h5^a^MX9YiA-1J}s4fq2VV9678x8@B2eN?3q3y99b_bHpaq z;5Z;aeYI*uq`$ttt9oyD|6G+lxG@=IRvP%RKQrln4eGpqCNyBz!c*&f#F-C5Oj!|{ z_5a59_RdC0|G#{-y|L2&ckx&o64M|%?qCME3lv|3Q2jj?@Uyb<%-HXxkWi&->~SIt zV@A(lj?+X*C4o3v;$b2e)g_!IO2{sLkSL+F)Kf$z{r6enWR6;@2pxXZuM!h|jBh?t ze1%D=}s~qytWvj<~rXEjfH(HAvW18)RQCogyKZ{g5?n^l8lOuIY z=X_>6ucM@)kdjS&2d$MD@5!X zxEMhgcHzcxI>4{|bOhJlp@cEg|Em|p)S-vG6D&t)um9=O73QBlh2-MXC#AANke@z9 zIuE>m`1GlEd;RGX^4>7+7Z|xU$7<8rAFVkIa~_8wQAR*3B(D0U+|MG1A^zfI1)6b^| zZ$F2x3RBlSi%#fCfwt#@td+4Qq zVILJW61@J)v*KK=Q{YtuTwe3t6R1)5$ZfzG>j^(PAfs+-8wg`6V)!BiZwN9uRU;&p z;V#6v&kPe;ZqymHdBOJKL-H-(5T{;(yO5 zldXe*&sR~vRTOX)1zbe|S5d%66a`#v2<`#I6q|w>rE~vTlUkmG*z13)I|*Su+yQA7 z99g9km@}Qg%+ZT-P60=en&bP%Usr#i?=B)0dwuQecRKu+l@4Imo!SFBw!aF0PTpT| z^1;Lw?{v~Vq&VNBh{8=zMv}-#V>XhGvy9Z%sw-V6v+}{5z4O&a9~FAKR!@^738N_LtPlV8s%KIJ|m zAy`Ll>C0*rVtg@I9qPN%Cr^+ZfdH+%1i~EBdkS?}BtQek^;M!+61}f~&^IK*Zc2k7+u+W{q5zKduNA8v)8&`6_o`#xB7Ww56@nk>E`lx z-w0&F{JS<8%M>|p$|2HP=-UeXjFpw7e`%7xB&nZ8;uq=WBCVN3f36O9*Vo`?X{x0a z&Mm{Y7p>D`R-Ot=?iFc&er{%}+_`!!+!GDX$0qJ735%vIl1+S8R?KowJYtO2Z9X&O zzh=(HhFrZf@qkPxi|_UJ`5CvIMwTO6V@GhRr#w0xBHN06Ro~$-8_CfO#-y=Gz7D zh&4Rj4j#B=m1c~{=LK!d%kaaroyQy^bSkBx2FfJL~Hrbr71Tz zO$Rp3Ymv`$JJZ(ccBa+sOszOv0iM1cz|-_gml`n{5hquFB!nRZjv28ScK>d!durNR zIX4=dMR=$;7=82;33S+3s{|(t`WtPO-rjvtMM42wIX@2Iiuh3BVs+V6vxb{$Av6pH zkLkKGIa1f_x)@;&!Mc$fYfP29#=SezaAcMxUQh@|E z4P6x}xEN5cwU{g+^IB6~OMur+A^#vUv{hM~7dwaaDHa~nV`_<=gqVU(?u-~WZ$4ll zf!p6`I6)F$f3_$^@%M1fonL0dg&WXMxKoEQCh$_ituEIy3gxZWgGMZ^$|D zvFH>bTZK_`lT3!k1-NSi1xtgyWS~03QCX$uY73c8bnui6Q4t!L?G3oSoOD0tgbvUA z5d^VP6Nf{}(w`4*z>l@7t+21C^kLPJs272Qn}~x6Y*ih(C3~Z(^^PckXQBieBg9#M z2;HJ6T54XTK6i!@fMX@f=-`Hkif=3(GDpo*_%01dYmUeIyTw_p>{#nDoqiarI`0hD znd_?W3EMf_g4PIMslThZXqznq(=vNq{aL@B8U1qCJhjHjY&v4zA>aVGr}ZRZwC-ku zQn3;y#I^V7wDvbQoACC_hU36Z;j31Ey*XD*Yn7V)KAtK5AAW&-uXqS>b}!Jh`=2)} z{-5pM*3QcRa~Dr(6;2jINYDi4sM!xO8Mm(9oDFo{uxcSx}_-kaQuqzrj0XQsaT7)s_OYgly)K{4V}MM zGlI9ezF;Y_Kl+9imwN5KbS0$`k!^J=p#wI*I&7(&%xH~gSxJ_%epG6IUl$U~c$rLm zZ;CUWcq^@0qlzA?%1kT-dG}h=ZA30+Jvz-`aNw* z66O67=&4d~+>ee@!%{ncWNGIl=auhRQ8n0OnN0s)(<@7m<=Hytds%?z5}7vA{pCv% zalP+Y$oKfn9{-y?3Fs93_g3%at5W=LbEmhpivQikGbR3arz?*JNjZ;?ZQQ|!O(R~u#fZ|l{@3jcK{k6j@#4eHMT-U&FY zM##PmFX;;EwNwY`D#muVF}CGbICpR%@8Me_JFSBPw`#7s^xK0nPwUSDF4b6V(DV1_ z4|g$TPHCivG;VQ!ExN~;(4C8XsaIDsph?(n*@ZsE{ba+9cKttlBw&jE?`?0D_5V(9V|%6l@8X%o|9^LjL)|X?ZkCFbo?AVuXZ3s! S&;Jbo0RR8+saxX!@Bsi(lkOw{ diff --git a/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml b/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml index fc78df006c..39790e1438 100644 --- a/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml +++ b/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml @@ -38,6 +38,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] verbs: ["get", "list"] diff --git a/deploy/rbac-csi-azurefile-node.yaml b/deploy/rbac-csi-azurefile-node.yaml index 72c87e5470..d7b10b2d7e 100644 --- a/deploy/rbac-csi-azurefile-node.yaml +++ b/deploy/rbac-csi-azurefile-node.yaml @@ -37,6 +37,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] verbs: ["get", "list"] @@ -54,24 +57,3 @@ roleRef: name: csi-azurefile-node-katacc-role apiGroup: rbac.authorization.k8s.io --- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-azurefile-node-role -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-azurefile-node-binding -subjects: - - kind: ServiceAccount - name: csi-azurefile-node-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-azurefile-node-role - apiGroup: rbac.authorization.k8s.io