diff --git a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.jar b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.jar index 8a6d07377..43294faea 100644 Binary files a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.jar and b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.jar differ diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 0bf2d5063..c57d3df20 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -32,34 +32,32 @@ services: db-updater: condition: service_started kafka: - condition: service_started + condition: service_healthy volumes: - logs_volume:/opt/clearing/bin/log_artifacts - zookeeper: - image: mirror.gcr.io/confluentinc/cp-zookeeper - container_name: zookeeper - environment: - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - kafka: - image: mirror.gcr.io/confluentinc/cp-kafka + image: mirror.gcr.io/bitnami/kafka:4.0.0 container_name: kafka environment: KAFKA_BROKER_ID: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_LOG4J_LOGGERS: kafka=WARN,kafka.controller=WARN,kafka.log.LogCleaner=WARN,state.change.logger=WARN,kafka.producer.async.DefaultEventHandler=WARN - KAFKA_LOG4J_ROOT_LOGLEVEL: WARN - depends_on: - - zookeeper + KAFKA_CFG_NODE_ID: 1 + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_ENABLE_KRAFT: yes + KAFKA_CFG_PROCESS_ROLES: broker,controller + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true + volumes: + - ./kafka/log4j2.yaml:/opt/bitnami/kafka/config/log4j2.yaml + healthcheck: + test: kafka-topics.sh --list --bootstrap-server kafka:9092 || exit 1 + interval: 1s + timeout: 60s + retries: 100 sftp: build: sftp diff --git a/clr_test/kafka/log4j2.yaml b/clr_test/kafka/log4j2.yaml new file mode 100644 index 000000000..f4f8739b3 --- /dev/null +++ b/clr_test/kafka/log4j2.yaml @@ -0,0 +1,135 @@ +Configuration: + Properties: + Property: + # Fallback if the system property is not set + - name: "kafka.logs.dir" + value: "." + - name: "logPattern" + value: "[%d] %p %m (%c)%n" + + # Appenders configuration + # See: https://logging.apache.org/log4j/2.x/manual/appenders.html + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + RollingFile: + - name: KafkaAppender + fileName: "${sys:kafka.logs.dir}/server.log" + filePattern: "${sys:kafka.logs.dir}/server.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # State Change appender + - name: StateChangeAppender + fileName: "${sys:kafka.logs.dir}/state-change.log" + filePattern: "${sys:kafka.logs.dir}/stage-change.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Request appender + - name: RequestAppender + fileName: "${sys:kafka.logs.dir}/kafka-request.log" + filePattern: "${sys:kafka.logs.dir}/kafka-request.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Cleaner appender + - name: CleanerAppender + fileName: "${sys:kafka.logs.dir}/log-cleaner.log" + filePattern: "${sys:kafka.logs.dir}/log-cleaner.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Controller appender + - name: ControllerAppender + fileName: "${sys:kafka.logs.dir}/controller.log" + filePattern: "${sys:kafka.logs.dir}/controller.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Authorizer appender + - name: AuthorizerAppender + fileName: "${sys:kafka.logs.dir}/kafka-authorizer.log" + filePattern: "${sys:kafka.logs.dir}/kafka-authorizer.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + + # Loggers configuration + # See: https://logging.apache.org/log4j/2.x/manual/configuration.html#configuring-loggers + Loggers: + Root: + level: WARN + AppenderRef: + - ref: STDOUT + - ref: KafkaAppender + Logger: + # Kafka logger + - name: kafka + level: WARN + # Kafka org.apache logger + - name: org.apache.kafka + level: WARN + # Kafka request logger + - name: kafka.request.logger + level: WARN + additivity: false + AppenderRef: + ref: RequestAppender + # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE + # for additional output related to the handling of requests + # - name: kafka.network.Processor + # level: TRACE + # additivity: false + # AppenderRef: + # ref: RequestAppender + # - name: kafka.server.KafkaApis + # level: TRACE + # additivity: false + # AppenderRef: + # ref: RequestAppender + # Kafka network RequestChannel$ logger + - name: kafka.network.RequestChannel$ + level: WARN + additivity: false + AppenderRef: + ref: RequestAppender + # Controller logger + - name: org.apache.kafka.controller + level: WARN + additivity: false + AppenderRef: + ref: ControllerAppender + # LogCleaner logger + - name: kafka.log.LogCleaner + level: WARN + additivity: false + AppenderRef: + ref: CleanerAppender + # State change logger + - name: state.change.logger + level: WARN + additivity: false + AppenderRef: + ref: StateChangeAppender + # Authorizer logger + - name: kafka.authorizer.logger + level: WARN + additivity: false + AppenderRef: + ref: AuthorizerAppender