diff --git a/pom.xml b/pom.xml
index e4a0e83..dc2df27 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,6 +38,11 @@
mybatis-spring-boot-starter
3.0.4
+
+ org.springframework.kafka
+ spring-kafka
+ 3.3.15
+
ru.nbch
pipeline-engine
diff --git a/src/main/java/ru/nbch/credit_tracker/config/kafka/KafkaConfig.java b/src/main/java/ru/nbch/credit_tracker/config/kafka/KafkaConfig.java
new file mode 100644
index 0000000..016734d
--- /dev/null
+++ b/src/main/java/ru/nbch/credit_tracker/config/kafka/KafkaConfig.java
@@ -0,0 +1,71 @@
+package ru.nbch.credit_tracker.config.kafka;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.kafka.annotation.EnableKafka;
+import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
+import org.springframework.kafka.core.*;
+import org.springframework.kafka.listener.ContainerProperties;
+import org.springframework.kafka.support.serializer.JsonDeserializer;
+import ru.nbch.credit_tracker.service.mode.monitoring.TestData;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@EnableKafka
+@Configuration
+public class KafkaConfig {
+
+ @Bean
+ public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory(ConsumerFactory consumerFactory) {
+ ConcurrentKafkaListenerContainerFactory factory =
+ new ConcurrentKafkaListenerContainerFactory<>();
+ factory.setConsumerFactory(consumerFactory);
+ factory.setBatchListener(true);
+ factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL);
+
+ return factory;
+ }
+
+ @Bean
+ public ConsumerFactory consumerFactory() {
+ Map props = new HashMap<>();
+ props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:29092");
+ props.put(ConsumerConfig.GROUP_ID_CONFIG, "credit-tracker");
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 20);
+
+ JsonDeserializer jsonDeserializer = new JsonDeserializer<>(TestData.class);
+ jsonDeserializer.addTrustedPackages("*");
+ jsonDeserializer.setUseTypeHeaders(false);
+
+
+ props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+
+ return new DefaultKafkaConsumerFactory<>(
+ props,
+ new StringDeserializer(),
+ jsonDeserializer
+ );
+ }
+
+ @Bean
+ public ProducerFactory producerFactory() {
+ Map props = new HashMap<>();
+ props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:29092");
+ props.put(ProducerConfig.LINGER_MS_CONFIG, 10);
+ props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+ props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+ return new DefaultKafkaProducerFactory<>(props);
+ }
+
+ @Bean
+ public KafkaTemplate kafkaTemplate(ProducerFactory producerFactory) {
+ return new KafkaTemplate<>(producerFactory);
+ }
+
+}
diff --git a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/CTMonitoringApplication.java b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/CTMonitoringApplication.java
index 9e6c42d..6e5257a 100644
--- a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/CTMonitoringApplication.java
+++ b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/CTMonitoringApplication.java
@@ -1,5 +1,6 @@
package ru.nbch.credit_tracker.service.mode.monitoring;
+import com.fasterxml.jackson.core.JsonProcessingException;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -21,11 +22,13 @@ import org.springframework.context.annotation.FilterType;
}
)
public class CTMonitoringApplication {
- public static void main(String[] args) {
+ public static void main(String[] args) throws JsonProcessingException {
SpringApplication application = new SpringApplication(CTMonitoringApplication.class);
application.setWebApplicationType(WebApplicationType.NONE);
ConfigurableApplicationContext ctx = application.run(args);
- MonitoringService monitoringService = ctx.getBean(MonitoringService.class);
- monitoringService.run();
+// MonitoringService monitoringService = ctx.getBean(MonitoringService.class);
+// monitoringService.run();
+
+// ctx.getBean(KafkaMonitoringService.class).generateMessages();
}
}
diff --git a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/KafkaMonitoringService.java b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/KafkaMonitoringService.java
new file mode 100644
index 0000000..62a4315
--- /dev/null
+++ b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/KafkaMonitoringService.java
@@ -0,0 +1,94 @@
+package ru.nbch.credit_tracker.service.mode.monitoring;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+
+@Slf4j
+@Service
+public class KafkaMonitoringService {
+
+ private final KafkaTemplate kafkaTemplate;
+ private final MonitoringTimerService monitoringTimerService;
+
+ @Value("${app.tmp-dir}")
+ private String tmpDir;
+
+ public KafkaMonitoringService(KafkaTemplate kafkaTemplate,
+ MonitoringTimerService monitoringTimerService) {
+ this.kafkaTemplate = kafkaTemplate;
+ this.monitoringTimerService = monitoringTimerService;
+ }
+
+ @KafkaListener(
+ topics = "my-topic",
+ concurrency = "3"
+ )
+ public void listen(List> records,
+ Acknowledgment acknowledgment) {
+ for (ConsumerRecord record : records) {
+ try {
+ processRecord(record.value());
+ } catch (Exception e) {
+ log.error("Failed to process record: {}", record, e);
+ }
+ }
+ // insert to db
+ acknowledgment.acknowledge();
+ }
+
+ private void processRecord(PackageMessage msg) throws IOException {
+ Path packageDir = Paths.get(tmpDir, String.valueOf(msg.packageId()));
+ Files.createDirectories(packageDir);
+
+ Path filePath = packageDir.resolve(msg.uuid() + ".xml");
+ Files.writeString(filePath, msg.message(), StandardOpenOption.CREATE_NEW);
+
+ monitoringTimerService.registerFile(msg.packageId(), filePath);
+ }
+
+ // TODO send PackageMessage
+ public void generateMessages() throws JsonProcessingException {
+ ObjectMapper mapper = JsonMapper.builder()
+ .build();
+ for (int i = 0; i < 500; i++) {
+ TestData data = new TestData();
+ data.setId(UUID.randomUUID());
+ data.setMessage("Message " + i);
+ send(data.getId().toString(), mapper.writeValueAsString(data));
+ }
+ }
+
+ private void send(String id, String data) {
+ CompletableFuture> future =
+ kafkaTemplate.send("test-topic", id, data);
+
+ future.whenComplete((result, exception) -> {
+ if (exception == null) {
+ log.info("Sent user event=[{}] with offset=[{}]",
+ data, result.getRecordMetadata().offset());
+ } else {
+ log.error("Unable to send user event=[{}] due to: {}",
+ data, exception.getMessage());
+ }
+ });
+ }
+
+}
diff --git a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringService.java b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringService.java
index 5f9eca5..b0361a7 100644
--- a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringService.java
+++ b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringService.java
@@ -3,7 +3,6 @@ package ru.nbch.credit_tracker.service.mode.monitoring;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.MDC;
-import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.entities.app.Hit;
@@ -37,9 +36,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -72,9 +68,9 @@ public class MonitoringService {
}
public void run() {
- ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-"));
- long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15;
- scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES);
+// ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-"));
+// long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15;
+// scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES);
}
private void monitor() {
diff --git a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringTimerService.java b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringTimerService.java
new file mode 100644
index 0000000..108445d
--- /dev/null
+++ b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringTimerService.java
@@ -0,0 +1,137 @@
+package ru.nbch.credit_tracker.service.mode.monitoring;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.*;
+
+@Service
+@Slf4j
+public class MonitoringTimerService {
+
+ private static final long DELAY_MINUTES = 5;
+
+ @Value("${app.target-dir}")
+ private String targetDir;
+
+ private final ScheduledExecutorService scheduler =
+ Executors.newScheduledThreadPool(4);
+
+ private final ConcurrentHashMap bundles =
+ new ConcurrentHashMap<>();
+
+ public void registerFile(Integer packageId, Path filePath) {
+ bundles.compute(packageId, (id, existing) -> {
+ if (existing == null || existing.isIdle()) {
+ PackageBundle bundle = new PackageBundle();
+ bundle.addFile(filePath);
+ scheduleFlush(packageId, bundle);
+ log.info("Started new timer for packageId={}", packageId);
+ return bundle;
+ }
+ existing.addFile(filePath);
+ log.debug("Queued file for packageId={} state={}", packageId, existing.getState());
+ return existing;
+ });
+ }
+
+ private void scheduleFlush(Integer packageId, PackageBundle bundle) {
+ bundle.setState(BundleState.SCHEDULED);
+ scheduler.schedule(() -> flush(packageId, bundle), DELAY_MINUTES, TimeUnit.MINUTES);
+ }
+
+ private void flush(Integer packageId, PackageBundle bundle) {
+ bundle.setState(BundleState.MERGING);
+ List snapshot = bundle.drainFiles();
+
+ try {
+ if (!snapshot.isEmpty()) {
+ Path destDir = Paths.get(targetDir, String.valueOf(packageId));
+ Files.createDirectories(destDir);
+
+ List movedFiles = new ArrayList<>();
+ for (Path src : snapshot) {
+ if (!Files.exists(src)) {
+ log.warn("File missing, skipping: {}", src);
+ continue;
+ }
+ Path dest = destDir.resolve(src.getFileName());
+ Files.move(src, dest, StandardCopyOption.ATOMIC_MOVE);
+ movedFiles.add(dest);
+ }
+ mergeXmlFiles(movedFiles, destDir, packageId);
+ }
+ } catch (IOException e) {
+ log.error("Flush failed for packageId={}", packageId, e);
+ } finally {
+ bundle.setState(BundleState.IDLE);
+ bundles.remove(packageId);
+
+ // Файлы, добавленные во время мержа — забираем и стартуем новый цикл
+ List leftover = bundle.drainFiles();
+ if (!leftover.isEmpty()) {
+ log.info("Files arrived during merge, restarting timer for packageId={}, count={}",
+ packageId, leftover.size());
+ PackageBundle next = new PackageBundle();
+ leftover.forEach(next::addFile);
+ bundles.put(packageId, next);
+ scheduleFlush(packageId, next);
+ }
+ }
+ }
+
+ private void mergeXmlFiles(List files, Path destDir, Integer packageId) throws IOException {
+ if (files.isEmpty()) return;
+ Path merged = destDir.resolve("merged_" + packageId + ".xml");
+ try (BufferedWriter writer = Files.newBufferedWriter(merged)) {
+ writer.write("");
+ writer.newLine();
+ for (Path file : files) {
+ writer.write(Files.readString(file));
+ writer.newLine();
+ }
+ writer.write("");
+ }
+ log.info("Merged {} files → {} for packageId={}", files.size(), merged, packageId);
+ }
+
+ private enum BundleState {SCHEDULED, MERGING, IDLE}
+
+ private static class PackageBundle {
+ private final Queue files = new ConcurrentLinkedQueue<>();
+ private volatile BundleState state = BundleState.IDLE;
+
+ void addFile(Path path) {
+ files.add(path);
+ }
+
+ List drainFiles() {
+ List result = new ArrayList<>();
+ Path p;
+ while ((p = files.poll()) != null) result.add(p);
+ return result;
+ }
+
+ BundleState getState() {
+ return state;
+ }
+
+ void setState(BundleState s) {
+ this.state = s;
+ }
+
+ boolean isIdle() {
+ return state == BundleState.IDLE;
+ }
+ }
+}
diff --git a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/PackageMessage.java b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/PackageMessage.java
new file mode 100644
index 0000000..21183f1
--- /dev/null
+++ b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/PackageMessage.java
@@ -0,0 +1,9 @@
+package ru.nbch.credit_tracker.service.mode.monitoring;
+
+import java.util.UUID;
+
+public record PackageMessage(
+ Integer packageId,
+ String message,
+ UUID uuid
+) {}
diff --git a/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/TestData.java b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/TestData.java
new file mode 100644
index 0000000..973e5e4
--- /dev/null
+++ b/src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/TestData.java
@@ -0,0 +1,17 @@
+package ru.nbch.credit_tracker.service.mode.monitoring;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.UUID;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class TestData {
+ private UUID id;
+ private String message;
+}