chore/kraft migration

deploy/helm/kafka-operator/crds/crds.yaml

@@ -731,9 +731,7 @@ spec:
                     authentication: []
                     authorization:
                       opa: null
-                    tls:
-                      internalSecretClass: tls
-                      serverSecretClass: tls
+                    metadataManager: ZooKeeper
                     zookeeperConfigMapName: null
                   description: |-
                     Kafka settings that affect all roles and role groups.
@@ -793,15 +791,17 @@ spec:
                             - configMapName
                           type: object
                       type: object
+                    metadataManager:
+                      default: ZooKeeper
+                      enum:
+                        - ZooKeeper
+                        - KRaft
+                      type: string
                     tls:
-                      default:
-                        internalSecretClass: tls
-                        serverSecretClass: tls
                       description: TLS encryption settings for Kafka (server, internal).
                       nullable: true
                       properties:
                         internalSecretClass:
-                          default: tls
                           description: |-
                             The [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass.html) to use for
                             internal broker communication. Use mutual verification between brokers (mandatory).
@@ -810,9 +810,9 @@ spec:
                             - Which ca.crt to use when validating the other brokers
 
                             Defaults to `tls`
+                          nullable: true
                           type: string
                         serverSecretClass:
-                          default: tls
                           description: |-
                             The [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass.html) to use for
                             client connections. This setting controls:
@@ -836,7 +836,7 @@ spec:
                         Provide the name of the ZooKeeper [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery)
                         here. When using the [Stackable operator for Apache ZooKeeper](https://docs.stackable.tech/home/nightly/zookeeper/)
                         to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource.
-                        This can only be used up to Kafka version 3.9.x. Since Kafka 4.0.0, ZooKeeper suppport was dropped.
+                        This can only be used up to Kafka version 3.9.x. Since Kafka 4.0.0, ZooKeeper support was dropped.
                         Please use the 'controller' role instead.
                       nullable: true
                       type: string

examples/kraft-migration/01-setup.yaml

@@ -0,0 +1,87 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    stackable.tech/vendor: Stackable
+  name: kraft-migration
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperCluster
+metadata:
+  name: simple-zk
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.8.3
+    pullPolicy: IfNotPresent
+  servers:
+    roleGroups:
+      default:
+        replicas: 3
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+  name: simple-kafka-znode
+  namespace: kraft-migration
+spec:
+  clusterRef:
+    name: simple-zk
+# ---
+# apiVersion: secrets.stackable.tech/v1alpha1
+# kind: SecretClass
+# metadata:
+#   name: kafka-internal-tls
+# spec:
+#   backend:
+#     autoTls:
+#       ca:
+#         secret:
+#           name: secret-provisioner-kafka-internal-tls-ca
+#           namespace: kraft-migration
+#         autoGenerate: true
+# ---
+# apiVersion: authentication.stackable.tech/v1alpha1
+# kind: AuthenticationClass
+# metadata:
+#   name: kafka-client-auth-tls
+# spec:
+#   provider:
+#     tls:
+#       clientCertSecretClass: kafka-client-auth-secret
+# ---
+# apiVersion: secrets.stackable.tech/v1alpha1
+# kind: SecretClass
+# metadata:
+#   name: kafka-client-auth-secret
+# spec:
+#   backend:
+#     autoTls:
+#       ca:
+#         secret:
+#           name: secret-provisioner-tls-kafka-client-ca
+#           namespace: kraft-migration
+#         autoGenerate: true
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: ZooKeeper
+    # authentication:
+    #   - authenticationClass: kafka-client-auth-tls
+    # tls:
+    #   internalSecretClass: kafka-internal-tls
+    #   serverSecretClass: tls
+    zookeeperConfigMapName: simple-kafka-znode
+  brokers:
+    roleGroups:
+      default:
+        replicas: 3

examples/kraft-migration/02-start-controllers.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: ZooKeeper
+    # authentication:
+    #   - authenticationClass: kafka-client-auth-tls
+    # tls:
+    #   internalSecretClass: kafka-internal-tls
+    #   serverSecretClass: tls
+    zookeeperConfigMapName: simple-kafka-znode
+  brokers:
+    envOverrides:
+      KAFKA_CLUSTER_ID: "lyeJYZ7TQ_SfT4HcU8W3iw"
+    roleGroups:
+      default:
+        replicas: 3
+  controllers:
+    roleGroups:
+      default:
+        replicas: 3
+    envOverrides:
+      KAFKA_CLUSTER_ID: "lyeJYZ7TQ_SfT4HcU8W3iw"
+    configOverrides:
+      controller.properties:
+        zookeeper.metadata.migration.enable: "true" # Enable migration mode so the controller can read metadata from ZooKeeper.

examples/kraft-migration/03-migrate-metadata.yaml

@@ -0,0 +1,47 @@
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: ZooKeeper
+    # authentication:
+    #   - authenticationClass: kafka-client-auth-tls
+    # tls:
+    #   internalSecretClass: kafka-internal-tls
+    #   serverSecretClass: tls
+    zookeeperConfigMapName: simple-kafka-znode
+  brokers:
+    envOverrides:
+      KAFKA_CLUSTER_ID: "lyeJYZ7TQ_SfT4HcU8W3iw"
+    roleGroups:
+      default:
+        replicas: 3
+    configOverrides:
+      broker.properties:
+        inter.broker.protocol.version: "3.9" # - Latest value known to Kafka 3.9.1
+        zookeeper.metadata.migration.enable: "true" # - Enable migration mode so the broker can participate in metadata migration.
+        controller.listener.names: "CONTROLLER"
+        controller.quorum.voters: "2110489703@simple-kafka-controller-default-0.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9092,2110489704@simple-kafka-controller-default-1.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9092,2110489705@simple-kafka-controller-default-2.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9092"
+
+  # listener.security.protocol.map: CONTROLLER:SSL,... - Already defined by the operator
+  # zookeeper.connect=<zk_connection_string> (should already be present) - The ZooKeeper connection string. This property should already be configured.
+  # controller.quorum.voters=<voter_string> (same as controllers) - Specify the same controller quorum voters string as configured in phase 2.
+  # controller.listener.names=CONTROLLER - Define the listener name for the controller.
+  # Add CONTROLLER to listener.security.protocol.map (for example, ...CONTROLLER:PLAINTEXT) - Add the CONTROLLER listener to the security protocol map with the appropriate security protocol.
+  # confluent.cluster.link.metadata.topic.enable=true - This property is used by Cluster Linking during the migration.
+
+  controllers:
+    roleGroups:
+      default:
+        replicas: 3
+    envOverrides:
+      KAFKA_CLUSTER_ID: "lyeJYZ7TQ_SfT4HcU8W3iw"
+    configOverrides:
+      controller.properties:
+        zookeeper.metadata.migration.enable: "true" # Enable migration mode so the controller can read metadata from ZooKeeper.

rust/operator-binary/src/config/command.rs

@@ -19,7 +19,6 @@ use crate::{
 /// Returns the commands to start the main Kafka container
 pub fn broker_kafka_container_commands(
     kafka: &v1alpha1::KafkaCluster,
-    cluster_id: &str,
     controller_descriptors: Vec<KafkaPodDescriptor>,
     kafka_security: &KafkaTlsSecurity,
     product_version: &str,
@@ -42,17 +41,16 @@ pub fn broker_kafka_container_commands(
             true => format!("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH})"),
             false => "".to_string(),
         },
-        broker_start_command = broker_start_command(kafka, cluster_id, controller_descriptors, product_version),
+        broker_start_command = broker_start_command(kafka, controller_descriptors, product_version),
     }
 }
 
 fn broker_start_command(
     kafka: &v1alpha1::KafkaCluster,
-    cluster_id: &str,
     controller_descriptors: Vec<KafkaPodDescriptor>,
     product_version: &str,
 ) -> String {
-    if kafka.is_controller_configured() {
+    if kafka.is_kraft_mode() {
         formatdoc! {"
             POD_INDEX=$(echo \"$POD_NAME\" | grep -oE '[0-9]+$')
             export REPLICA_ID=$((POD_INDEX+NODE_ID_OFFSET))
@@ -63,7 +61,7 @@ fn broker_start_command(
             cp {config_dir}/jaas.properties /tmp/jaas.properties
             config-utils template /tmp/jaas.properties
 
-            bin/kafka-storage.sh format --cluster-id {cluster_id} --config /tmp/{properties_file} --ignore-formatted {initial_controller_command}
+            bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted {initial_controller_command}
             bin/kafka-server-start.sh /tmp/{properties_file} &
         ",
         config_dir = STACKABLE_CONFIG_DIR,
@@ -72,6 +70,9 @@ fn broker_start_command(
         }
     } else {
         formatdoc! {"
+            POD_INDEX=$(echo \"$POD_NAME\" | grep -oE '[0-9]+$')
+            export REPLICA_ID=$((POD_INDEX+NODE_ID_OFFSET))
+
             cp {config_dir}/{properties_file} /tmp/{properties_file}
             config-utils template /tmp/{properties_file}
 
@@ -128,7 +129,6 @@ wait_for_termination()
 "#;
 
 pub fn controller_kafka_container_command(
-    cluster_id: &str,
     controller_descriptors: Vec<KafkaPodDescriptor>,
     product_version: &str,
 ) -> String {
@@ -145,7 +145,7 @@ pub fn controller_kafka_container_command(
 
         config-utils template /tmp/{properties_file}
 
-        bin/kafka-storage.sh format --cluster-id {cluster_id} --config /tmp/{properties_file} --ignore-formatted {initial_controller_command}
+        bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted {initial_controller_command}
         bin/kafka-server-start.sh /tmp/{properties_file} &
 
         wait_for_termination $!

rust/operator-binary/src/crd/listener.rs

@@ -401,7 +401,7 @@ mod tests {
                     ),
                 },
             }]),
-            "internalTls".to_string(),
+            Some("internalTls".to_string()),
             Some("tls".to_string()),
         );
         let cluster_info = default_cluster_info();
@@ -460,7 +460,7 @@ mod tests {
 
         let kafka_security = KafkaTlsSecurity::new(
             ResolvedAuthenticationClasses::new(vec![]),
-            "tls".to_string(),
+            Some("tls".to_string()),
             Some("tls".to_string()),
         );
         let config =
@@ -514,11 +514,8 @@ mod tests {
             )
         );
 
-        let kafka_security = KafkaTlsSecurity::new(
-            ResolvedAuthenticationClasses::new(vec![]),
-            "".to_string(),
-            None,
-        );
+        let kafka_security =
+            KafkaTlsSecurity::new(ResolvedAuthenticationClasses::new(vec![]), None, None);
 
         let config =
             get_kafka_listener_config(&kafka, &kafka_security, &rolegroup_ref, &cluster_info)
@@ -603,7 +600,7 @@ mod tests {
                     ),
                 },
             }]),
-            "tls".to_string(),
+            Some("tls".to_string()),
             Some("tls".to_string()),
         );
         let cluster_info = default_cluster_info();