zookeeper removed, clearing-tester.jar updated

clr_test/docker-compose.yml

@@ -32,34 +32,32 @@ services:
       db-updater:
         condition: service_started
       kafka:
-        condition: service_started
+        condition: service_healthy
     volumes:
       - logs_volume:/opt/clearing/bin/log_artifacts
 
-  zookeeper:
-    image: mirror.gcr.io/confluentinc/cp-zookeeper
-    container_name: zookeeper
-    environment:
-      ZOOKEEPER_SERVER_ID: 1
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-
   kafka:
-    image: mirror.gcr.io/confluentinc/cp-kafka
+    image: mirror.gcr.io/bitnami/kafka:4.0.0
     container_name: kafka
     environment:
       KAFKA_BROKER_ID: 1
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_CFG_NODE_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+      KAFKA_ENABLE_KRAFT: yes
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_CFG_PROCESS_ROLES: broker,controller
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
-      KAFKA_LOG4J_LOGGERS: kafka=WARN,kafka.controller=WARN,kafka.log.LogCleaner=WARN,state.change.logger=WARN,kafka.producer.async.DefaultEventHandler=WARN
+      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
-      KAFKA_LOG4J_ROOT_LOGLEVEL: WARN
+      ALLOW_PLAINTEXT_LISTENER: yes
-    depends_on:
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
-      - zookeeper
+    volumes:
+      - ./kafka/log4j2.yaml:/opt/bitnami/kafka/config/log4j2.yaml
+    healthcheck:
+      test: kafka-topics.sh --list --bootstrap-server kafka:9092 || exit 1
+      interval: 1s
+      timeout: 60s
+      retries: 100
 
   sftp:
     build: sftp

clr_test/kafka/log4j2.yaml

@@ -0,0 +1,135 @@
+Configuration:
+  Properties:
+    Property:
+      # Fallback if the system property is not set
+      - name: "kafka.logs.dir"
+        value: "."
+      - name: "logPattern"
+        value: "[%d] %p %m (%c)%n"
+
+  # Appenders configuration
+  # See: https://logging.apache.org/log4j/2.x/manual/appenders.html
+  Appenders:
+    Console:
+      name: STDOUT
+      PatternLayout:
+        pattern: "${logPattern}"
+
+    RollingFile:
+      - name: KafkaAppender
+        fileName: "${sys:kafka.logs.dir}/server.log"
+        filePattern: "${sys:kafka.logs.dir}/server.log.%d{yyyy-MM-dd-HH}"
+        PatternLayout:
+          pattern: "${logPattern}"
+        TimeBasedTriggeringPolicy:
+          modulate: true
+          interval: 1
+      # State Change appender
+      - name: StateChangeAppender
+        fileName: "${sys:kafka.logs.dir}/state-change.log"
+        filePattern: "${sys:kafka.logs.dir}/stage-change.log.%d{yyyy-MM-dd-HH}"
+        PatternLayout:
+          pattern: "${logPattern}"
+        TimeBasedTriggeringPolicy:
+          modulate: true
+          interval: 1
+      # Request appender
+      - name: RequestAppender
+        fileName: "${sys:kafka.logs.dir}/kafka-request.log"
+        filePattern: "${sys:kafka.logs.dir}/kafka-request.log.%d{yyyy-MM-dd-HH}"
+        PatternLayout:
+          pattern: "${logPattern}"
+        TimeBasedTriggeringPolicy:
+          modulate: true
+          interval: 1
+      # Cleaner appender
+      - name: CleanerAppender
+        fileName: "${sys:kafka.logs.dir}/log-cleaner.log"
+        filePattern: "${sys:kafka.logs.dir}/log-cleaner.log.%d{yyyy-MM-dd-HH}"
+        PatternLayout:
+          pattern: "${logPattern}"
+        TimeBasedTriggeringPolicy:
+          modulate: true
+          interval: 1
+      # Controller appender
+      - name: ControllerAppender
+        fileName: "${sys:kafka.logs.dir}/controller.log"
+        filePattern: "${sys:kafka.logs.dir}/controller.log.%d{yyyy-MM-dd-HH}"
+        PatternLayout:
+          pattern: "${logPattern}"
+        TimeBasedTriggeringPolicy:
+          modulate: true
+          interval: 1
+      # Authorizer appender
+      - name: AuthorizerAppender
+        fileName: "${sys:kafka.logs.dir}/kafka-authorizer.log"
+        filePattern: "${sys:kafka.logs.dir}/kafka-authorizer.log.%d{yyyy-MM-dd-HH}"
+        PatternLayout:
+          pattern: "${logPattern}"
+        TimeBasedTriggeringPolicy:
+          modulate: true
+          interval: 1
+
+  # Loggers configuration
+  # See: https://logging.apache.org/log4j/2.x/manual/configuration.html#configuring-loggers
+  Loggers:
+    Root:
+      level: WARN
+      AppenderRef:
+        - ref: STDOUT
+        - ref: KafkaAppender
+    Logger:
+      # Kafka logger
+      - name: kafka
+        level: WARN
+      # Kafka org.apache logger
+      - name: org.apache.kafka
+        level: WARN
+      # Kafka request logger
+      - name: kafka.request.logger
+        level: WARN
+        additivity: false
+        AppenderRef:
+          ref: RequestAppender
+      # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE
+      # for additional output related to the handling of requests
+      #      - name: kafka.network.Processor
+      #        level: TRACE
+      #        additivity: false
+      #        AppenderRef:
+      #          ref: RequestAppender
+      #      - name: kafka.server.KafkaApis
+      #        level: TRACE
+      #        additivity: false
+      #        AppenderRef:
+      #          ref: RequestAppender
+      # Kafka network RequestChannel$ logger
+      - name: kafka.network.RequestChannel$
+        level: WARN
+        additivity: false
+        AppenderRef:
+          ref: RequestAppender
+      # Controller logger
+      - name: org.apache.kafka.controller
+        level: WARN
+        additivity: false
+        AppenderRef:
+          ref: ControllerAppender
+      # LogCleaner logger
+      - name: kafka.log.LogCleaner
+        level: WARN
+        additivity: false
+        AppenderRef:
+          ref: CleanerAppender
+      # State change logger
+      - name: state.change.logger
+        level: WARN
+        additivity: false
+        AppenderRef:
+          ref: StateChangeAppender
+      # Authorizer logger
+      - name: kafka.authorizer.logger
+        level: WARN
+        additivity: false
+        AppenderRef:
+          ref: AuthorizerAppender