Implement remote_servers fragmentation via ConfigMaps

config/config-dev.yaml

@@ -85,6 +85,18 @@ clickhouse:
       # Default host_regexp to limit network connectivity from outside
       hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+    ################################################
+    ##
+    ## Configuration remote_servers section
+    ##
+    ################################################
+    remoteServers:
+      # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+      # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+      remoteServersSplitThresholdBytes: 614400
+      # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+      maxRemoteServersFragments: 100
+
   ################################################
   ##
   ## Configuration restart policy section

config/config.yaml

@@ -83,6 +83,18 @@ clickhouse:
       # Default host_regexp to limit network connectivity from outside
       hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+    ################################################
+    ##
+    ## Configuration remote_servers section
+    ##
+    ################################################
+    remoteServers:
+      # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+      # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+      remoteServersSplitThresholdBytes: 614400
+      # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+      maxRemoteServersFragments: 100
+
   ################################################
   ##
   ## Configuration restart policy section

deploy/builder/templates-config/config.yaml

@@ -77,6 +77,18 @@ clickhouse:
       # Default host_regexp to limit network connectivity from outside
       hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+    ################################################
+    ##
+    ## Configuration remote_servers section
+    ##
+    ################################################
+    remoteServers:
+      # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+      # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+      remoteServersSplitThresholdBytes: 614400
+      # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+      maxRemoteServersFragments: 100
+
   ################################################
   ##
   ## Configuration restart policy section

deploy/operator/clickhouse-operator-install-ansible.yaml

@@ -4901,6 +4901,18 @@ data:
           # Default host_regexp to limit network connectivity from outside
           hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+        ################################################
+        ##
+        ## Configuration remote_servers section
+        ##
+        ################################################
+        remoteServers:
+          # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+          # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+          remoteServersSplitThresholdBytes: 614400
+          # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+          maxRemoteServersFragments: 100
+
       ################################################
       ##
       ## Configuration restart policy section

deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml

@@ -5100,6 +5100,18 @@ data:
           # Default host_regexp to limit network connectivity from outside
           hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+        ################################################
+        ##
+        ## Configuration remote_servers section
+        ##
+        ################################################
+        remoteServers:
+          # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+          # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+          remoteServersSplitThresholdBytes: 614400
+          # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+          maxRemoteServersFragments: 100
+
       ################################################
       ##
       ## Configuration restart policy section

deploy/operator/clickhouse-operator-install-bundle.yaml

@@ -5160,6 +5160,18 @@ data:
           # Default host_regexp to limit network connectivity from outside
           hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+        ################################################
+        ##
+        ## Configuration remote_servers section
+        ##
+        ################################################
+        remoteServers:
+          # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+          # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+          remoteServersSplitThresholdBytes: 614400
+          # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+          maxRemoteServersFragments: 100
+
       ################################################
       ##
       ## Configuration restart policy section

deploy/operator/clickhouse-operator-install-template-v1beta1.yaml

@@ -4847,6 +4847,18 @@ data:
           # Default host_regexp to limit network connectivity from outside
           hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+        ################################################
+        ##
+        ## Configuration remote_servers section
+        ##
+        ################################################
+        remoteServers:
+          # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+          # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+          remoteServersSplitThresholdBytes: 614400
+          # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+          maxRemoteServersFragments: 100
+
       ################################################
       ##
       ## Configuration restart policy section

deploy/operator/clickhouse-operator-install-template.yaml

@@ -4894,6 +4894,18 @@ data:
           # Default host_regexp to limit network connectivity from outside
           hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+        ################################################
+        ##
+        ## Configuration remote_servers section
+        ##
+        ################################################
+        remoteServers:
+          # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+          # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+          remoteServersSplitThresholdBytes: 614400
+          # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+          maxRemoteServersFragments: 100
+
       ################################################
       ##
       ## Configuration restart policy section

deploy/operator/clickhouse-operator-install-tf.yaml

@@ -4901,6 +4901,18 @@ data:
           # Default host_regexp to limit network connectivity from outside
           hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
 
+        ################################################
+        ##
+        ## Configuration remote_servers section
+        ##
+        ################################################
+        remoteServers:
+          # Split remote_servers XML into multiple ConfigMaps when payload exceeds this threshold.
+          # 600 KiB payload plus XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+          remoteServersSplitThresholdBytes: 614400
+          # Maximum number of remote_servers fragments allowed for a CHI reconcile cycle.
+          maxRemoteServersFragments: 100
+
       ################################################
       ##
       ## Configuration restart policy section

docs/operator_configuration.md

@@ -9,11 +9,12 @@ Common configuration typically contains general ClickHouse configuration section
 1. ClickHouse user configuration files - ready-to-use XML files with sections of ClickHouse configuration **as-is**
 User configuration typically contains ClickHouse configuration sections with user accounts specifications. Those are exposed via config maps as well.
 1. `ClickHouseOperatorConfiguration` resource.
-1. `ClickHouseInstallationTemplate`s. Operator provides functionality to specify parts of `ClickHouseInstallation` manifest as a set of templates, which would be used in all `ClickHouseInstallation`s.   
+1. `ClickHouseInstallationTemplate`s. Operator provides functionality to specify parts of `ClickHouseInstallation` manifest as a set of templates, which would be used in all `ClickHouseInstallation`s. 
 
 ## Operator settings
 
 Operator settings are initialized in-order from 3 sources:
+
 * `/etc/clickhouse-operator/config.yaml`
 * etc-clickhouse-operator-files configmap (also a part of default [clickhouse-operator-install-bundle.yaml][clickhouse-operator-install-bundle.yaml]
 * `ClickHouseOperatorConfiguration` resource. See [example][70-chop-config.yaml] for details.
@@ -115,13 +116,42 @@ chPassword: clickhouse_operator_password
 chPort: 8123
 ```
 
+## `remote_servers` fragmentation
+
+Large CHIs can generate a very large `remote_servers.xml`. The operator can split this into deterministic fragments to keep ConfigMaps within Kubernetes size limits.
+
+Fragmentation knobs are configured under `clickhouse.configuration.remoteServers`:
+
+```yaml
+clickhouse:
+  configuration:
+    remoteServers:
+      # Default: 600 * 1024
+      # 600 KiB payload + XML metadata/wrappers keeps ConfigMap data safely below 1 MiB.
+      remoteServersSplitThresholdBytes: 614400
+
+      # Default: 100
+      maxRemoteServersFragments: 100
+```
+
+Behavior:
+
+1. If generated `remote_servers` fits into one fragment under threshold, the legacy file `chop-generated-remote_servers.xml` remains in `chi-<chi>-common-configd`.
+1. If threshold is exceeded, operator creates per-cluster fragment ConfigMaps: `chi-<chi>-<cluster>-remote-servers-shard-<paddedStart>`.
+1. Fragment file names are `chop-generated-remote_servers-part-<paddedStart>.xml`.
+1. StatefulSet pod template annotation `clickhouse.altinity.com/remote-servers-hash` is updated from canonical fragment metadata, so relevant pods roll out deterministically.
+1. During legacy fallback, stale fragment ConfigMaps are removed.
+
+ClickHouse merges all files from `config.d`, so multiple remote-server fragment files are applied automatically.
+
 ## ClickHouse Installation settings
 
 Operator deploys ClickHouse clusters with different defaults, that can be configured in a flexible way. 
 
 ### Default ClickHouse configuration files
 
 Default ClickHouse configuration files can be found in the following config maps, that are mounted to corresponding configuration folders of ClickHouse pods:
+
 * etc-clickhouse-operator-confd-files
 * etc-clickhouse-operator-configd-files
 * etc-clickhouse-operator-usersd-files
@@ -131,6 +161,7 @@ Config maps are initialized in default [clickhouse-operator-install-bundle.yaml]
 ### Defaults for ClickHouseInstallation
 
 Defaults for ClickHouseInstallation can be provided by `ClickHouseInstallationTemplate` it a variety of ways:
+
 * etc-clickhouse-operator-templatesd-files configmap
 * `ClickHouseInstallationTemplate` resources.
 
@@ -153,6 +184,7 @@ spec:
 ```
 
 Template needs to be deployed to some namespace, and later on used in the installation:
+
 ```
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseInstallation"

pkg/apis/clickhouse.altinity.com/v1/type_attributes.go

@@ -18,10 +18,17 @@ import core "k8s.io/api/core/v1"
 
 // ComparableAttributes specifies CHI attributes that are comparable
 type ComparableAttributes struct {
-	additionalEnvVars      []core.EnvVar      `json:"-" yaml:"-"`
-	additionalVolumes      []core.Volume      `json:"-" yaml:"-"`
-	additionalVolumeMounts []core.VolumeMount `json:"-" yaml:"-"`
-	skipOwnerRef           bool               `json:"-" yaml:"-"`
+	additionalEnvVars                []core.EnvVar        `json:"-" yaml:"-"`
+	additionalVolumes                []core.Volume        `json:"-" yaml:"-"`
+	additionalVolumeMounts           []core.VolumeMount   `json:"-" yaml:"-"`
+	remoteServersMounts              []RemoteServersMount `json:"-" yaml:"-"`
+	additionalPodTemplateAnnotations map[string]string    `json:"-" yaml:"-"`
+	skipOwnerRef                     bool                 `json:"-" yaml:"-"`
+}
+
+type RemoteServersMount struct {
+	ConfigMapName string
+	FileName      string
 }
 
 func (a *ComparableAttributes) GetAdditionalEnvVars() []core.EnvVar {
@@ -136,6 +143,44 @@ func (a *ComparableAttributes) GetSkipOwnerRef() bool {
 	return a.skipOwnerRef
 }
 
+func (a *ComparableAttributes) SetRemoteServersMounts(mounts []RemoteServersMount) {
+	if a == nil {
+		return
+	}
+	a.remoteServersMounts = mounts
+}
+
+func (a *ComparableAttributes) GetRemoteServersMounts() []RemoteServersMount {
+	if a == nil {
+		return nil
+	}
+	return a.remoteServersMounts
+}
+
+func (a *ComparableAttributes) SetAdditionalPodTemplateAnnotation(key, value string) {
+	if a == nil {
+		return
+	}
+	if a.additionalPodTemplateAnnotations == nil {
+		a.additionalPodTemplateAnnotations = make(map[string]string)
+	}
+	a.additionalPodTemplateAnnotations[key] = value
+}
+
+func (a *ComparableAttributes) DeleteAdditionalPodTemplateAnnotation(key string) {
+	if a == nil || a.additionalPodTemplateAnnotations == nil {
+		return
+	}
+	delete(a.additionalPodTemplateAnnotations, key)
+}
+
+func (a *ComparableAttributes) GetAdditionalPodTemplateAnnotations() map[string]string {
+	if a == nil {
+		return nil
+	}
+	return a.additionalPodTemplateAnnotations
+}
+
 func (a *ComparableAttributes) SetSkipOwnerRef(skip bool) {
 	if a == nil {
 		return

pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go

@@ -81,6 +81,11 @@ const (
 	// defaultMetricsTablesRegexp specifies default regexp to match tables in system database to fetch metrics from
 	defaultMetricsTablesRegexp = "^(metrics|custom_metrics)$"
 
+	// Default remote_servers fragmentation settings.
+	// 600KiB payload + XML metadata/wrappers keeps resulting ConfigMap data safely below 1MiB limit.
+	defaultRemoteServersSplitThresholdBytes = 600 * 1024
+	defaultRemoteServersMaxFragments        = 100
+
 	// defaultReconcileCHIsThreadsNumber specifies default number of controller threads running concurrently.
 	// Used in case no other specified in config
 	defaultReconcileCHIsThreadsNumber = 1
@@ -225,11 +230,29 @@ type OperatorConfigConfig struct {
 
 	User OperatorConfigUser `json:"user" yaml:"user"`
 
+	RemoteServers OperatorConfigRemoteServers `json:"remoteServers" yaml:"remoteServers"`
+
 	Network struct {
 		HostRegexpTemplate string `json:"hostRegexpTemplate" yaml:"hostRegexpTemplate"`
 	} `json:"network" yaml:"network"`
 }
 
+// OperatorConfigRemoteServers specifies remote_servers fragmentation behavior.
+type OperatorConfigRemoteServers struct {
+	RemoteServersSplitThresholdBytes int `json:"remoteServersSplitThresholdBytes" yaml:"remoteServersSplitThresholdBytes"`
+	MaxRemoteServersFragments        int `json:"maxRemoteServersFragments" yaml:"maxRemoteServersFragments"`
+}
+
+// SplitThresholdBytes returns configured remote_servers fragment payload threshold in bytes.
+func (c OperatorConfigConfig) SplitThresholdBytes() int {
+	return c.RemoteServers.RemoteServersSplitThresholdBytes
+}
+
+// MaxFragments returns configured maximum allowed number of remote_servers fragments.
+func (c OperatorConfigConfig) MaxFragments() int {
+	return c.RemoteServers.MaxRemoteServersFragments
+}
+
 // OperatorConfigRestartPolicyRuleSet specifies set of rules
 type OperatorConfigRestartPolicyRuleSet map[types.Matchable]types.StringBool
 
@@ -1079,6 +1102,15 @@ func (c *OperatorConfig) normalizeSectionClickHouseConfigurationUserDefault() {
 	// chConfigNetworksHostRegexpTemplate
 }
 
+func (c *OperatorConfig) normalizeSectionClickHouseConfigurationRemoteServers() {
+	if c.ClickHouse.Config.RemoteServers.RemoteServersSplitThresholdBytes <= 0 {
+		c.ClickHouse.Config.RemoteServers.RemoteServersSplitThresholdBytes = defaultRemoteServersSplitThresholdBytes
+	}
+	if c.ClickHouse.Config.RemoteServers.MaxRemoteServersFragments <= 0 {
+		c.ClickHouse.Config.RemoteServers.MaxRemoteServersFragments = defaultRemoteServersMaxFragments
+	}
+}
+
 func (c *OperatorConfig) normalizeSectionClickHouseAccess() {
 	// Username and Password to be used by operator to connect to ClickHouse instances for
 	// 1. Metrics requests
@@ -1199,6 +1231,7 @@ func (c *OperatorConfig) normalize() {
 
 	c.normalizeSectionClickHouseConfigurationFile()
 	c.normalizeSectionClickHouseConfigurationUserDefault()
+	c.normalizeSectionClickHouseConfigurationRemoteServers()
 	c.normalizeSectionClickHouseAccess()
 	c.normalizeSectionClickHouseMetrics()
 	c.normalizeSectionKeeperConfigurationFile()