From fc3fc9cef6e17d523e590c7e8907037f7513dc84 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 13 Sep 2024 11:37:53 +0300 Subject: [PATCH] xml-importer minor bug fix and refactoring --- .../importer/config/ImporterImdgConfig.java | 4 ++-- .../xml/importer/config/KafkaConfig.java | 21 ++++++++++--------- .../xml/importer/config/SFTPConfig.java | 2 +- .../importer/config/XMLImporterConfig.java | 11 ++++------ .../xml/importer/logic/steps/ImportToDB.java | 3 ++- .../logic/steps/XmlImportKafkaMessenger.java | 13 ++++-------- .../xml/importer/services/FileChecker.java | 1 + .../config/ImporterImdgTestConfig.java | 4 ++-- .../xml/importer/config/KafkaTestConfig.java | 10 +++------ .../importer/logic/steps/ImportToDBTest.java | 2 +- 10 files changed, 31 insertions(+), 40 deletions(-) diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java index 0c54ab12d..075b64d4a 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java @@ -32,8 +32,8 @@ public class ImporterImdgConfig { } @Autowired - @Bean - public HazelcastService imdgProvider( + @Bean(name = "hazelcastService") + public HazelcastService hazelcastService( @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, ImportXMLServiceSettings settings diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/KafkaConfig.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/KafkaConfig.java index 77c196d6c..1d3e64109 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/KafkaConfig.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/KafkaConfig.java @@ -3,6 +3,7 @@ package ru.spcex.clearing.xml.importer.config; import java.util.function.Supplier; import org.apache.kafka.clients.consumer.Consumer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -15,26 +16,26 @@ import ru.spcex.clearing.platform.messaging.config.element.KafkaProducerSettings import ru.spcex.clearing.platform.messaging.service.sender.KafkaSender; import ru.spcex.clearing.xml.importer.config.settings.ImportXMLServiceSettings; import ru.spcex.platform.imdg.api.ImdgId; -import ru.spcex.platform.imdg.api.ImdgProvider; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; @Configuration public class KafkaConfig { - @Bean - public ProducerFactory pf(ImportXMLServiceSettings settings) { + @Bean(name = "producerFactory") + public ProducerFactory producerFactory(ImportXMLServiceSettings settings) { KafkaProducerSettings kafkaSettings = settings.getKafkaProducer(); return KafkaProducerFactory.producerFactory(kafkaSettings); } - @Bean - public KafkaTemplate kafkaTemplate(ProducerFactory pf) { + @Bean(name = "kafkaTemplate") + public KafkaTemplate kafkaTemplate(@Qualifier("producerFactory") ProducerFactory pf) { return new KafkaTemplate<>(pf); } @Autowired - @Bean - public Supplier kafkaSender(KafkaTemplate kafkaTemplate, - ImdgProvider imdgProvider) { - ImdgId imdgIdGenerator = imdgProvider.getImdgIdGenerator(); + @Bean(name = "kafkaSender") + public Supplier kafkaSender(@Qualifier("kafkaTemplate") KafkaTemplate kafkaTemplate, + @Qualifier("hazelcastService") HazelcastService hazelcastService) { + ImdgId imdgIdGenerator = hazelcastService.getImdgIdGenerator(); return () -> KafkaSender .setup() .setKafkaTemplate(kafkaTemplate) @@ -45,7 +46,7 @@ public class KafkaConfig { @Autowired @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) - @Bean + @Bean(name = "createConsumer") public Consumer createConsumer(ImportXMLServiceSettings settings) { return KafkaConsumerFactory.consumer(settings.getKafkaConsumer()); } diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/SFTPConfig.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/SFTPConfig.java index 6b8a3a8d6..eb05d8a31 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/SFTPConfig.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/SFTPConfig.java @@ -99,7 +99,7 @@ public class SFTPConfig { @ConditionalOnProperty(value = "import-xml-service.process-sdf-files", havingValue = "true") @ServiceActivator(inputChannel = "listSftpChannelSdf") public MessageHandler handlerListSdf(@Qualifier("sftpSessionFactorySdf") SessionFactory sessionFactory, - ImportXMLServiceSettings settings) { + ImportXMLServiceSettings settings) { SftpOutboundGateway sftpOutboundGateway = new SftpOutboundGateway(sessionFactory, MGET.getCommand(), null); sftpOutboundGateway.setLocalDirectory(new File(settings.getStoreSdf().getSrcDir())); sftpOutboundGateway.setAutoCreateLocalDirectory(true); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java index 0a7f74107..bbde9098d 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java @@ -13,9 +13,9 @@ import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -29,22 +29,19 @@ import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.config.settings.ImportXMLServiceSettings; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; import ru.spcex.platform.classes.base.SpcexObjectBase; -import ru.spcex.platform.imdg.api.ImdgProvider; import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; @Configuration @EnableConfigurationProperties public class XMLImporterConfig { private final Logger log = LoggerFactory.getLogger(getClass()); private final ImportXMLServiceSettings settings; - private final ApplicationContext context; - private final ImdgProvider hazelcastService; + private final HazelcastService hazelcastService; public XMLImporterConfig(ImportXMLServiceSettings settings, - ApplicationContext context, - ImdgProvider hazelcastService) { + @Qualifier("hazelcastService") HazelcastService hazelcastService) { this.settings = settings; - this.context = context; this.hazelcastService = hazelcastService; } diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java index 545a61ef6..17aaa19ea 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java @@ -2,6 +2,7 @@ package ru.spcex.clearing.xml.importer.logic.steps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import ru.clearing.classes.statics.data.sdf.SDf57; import ru.spcex.clearing.platform.messaging.domain.cud.account.BankAccountNewRequest; @@ -25,7 +26,7 @@ public class ImportToDB { private final HazelcastService hazelcastService; private final XmlImportKafkaMessenger kafkaMessenger; - public ImportToDB(HazelcastService hazelcastService, + public ImportToDB(@Qualifier("hazelcastService") HazelcastService hazelcastService, XmlImportKafkaMessenger kafkaMessenger) { this.hazelcastService = hazelcastService; this.kafkaMessenger = kafkaMessenger; diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java index bc54f1f82..374c785b8 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java @@ -2,13 +2,14 @@ package ru.spcex.clearing.xml.importer.logic.steps; import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; -import java.util.HashMap; +import java.util.EnumMap; import java.util.Map; import java.util.function.Consumer; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import ru.spcex.clearing.platform.messaging.domain.Consts; import ru.spcex.clearing.platform.messaging.domain.cud.account.BankAccountNewRequest; @@ -18,7 +19,6 @@ import ru.spcex.clearing.platform.messaging.domain.cud.utilities.NotificationNew import ru.spcex.clearing.platform.messaging.domain.cud.utilities.PairSdfRequest; import ru.spcex.clearing.platform.messaging.serialization.LogFormatter; import ru.spcex.clearing.platform.messaging.service.sender.KafkaSender; -import ru.spcex.clearing.xml.importer.config.settings.ImportXMLServiceSettings; import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult; @@ -30,15 +30,10 @@ import ru.spcex.platform.enumeration.SdfTable; public class XmlImportKafkaMessenger implements InitializingBean { final Logger log = LoggerFactory.getLogger(getClass()); private final Supplier kafka; - private final Map> messengers; + private final Map> messengers = new EnumMap<>(ETable.class); - private final ImportXMLServiceSettings settings; - - public XmlImportKafkaMessenger(Supplier kafka, - ImportXMLServiceSettings settings) { + public XmlImportKafkaMessenger(@Qualifier("kafkaSender") Supplier kafka) { this.kafka = kafka; - this.settings = settings; - this.messengers = new HashMap<>(); } @Override diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/FileChecker.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/FileChecker.java index c53823314..8f602b73c 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/FileChecker.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/FileChecker.java @@ -57,6 +57,7 @@ public class FileChecker { List xmlFiles = srcDir.stream() .map(this::lsXML) .flatMap(List::stream) + .distinct() .toList(); if (xmlFiles.isEmpty()) return newFiles; diff --git a/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/ImporterImdgTestConfig.java b/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/ImporterImdgTestConfig.java index 7327c8829..9bb4d015d 100644 --- a/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/ImporterImdgTestConfig.java +++ b/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/ImporterImdgTestConfig.java @@ -64,8 +64,8 @@ public class ImporterImdgTestConfig { } @Autowired - @Bean(name = "imdgProvider") - public HazelcastService imdgTestProvider( + @Bean(name = "hazelcastService") + public HazelcastService hazelcastServiceTest( @Qualifier("taskExecutorHazelcastTestClientInitializerXmlImporter") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, @Qualifier("taskExecutorTestIdGeneratorAwaiterXmlImporter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, @Qualifier("hazelcastClientParamsXmlImporter") HazelcastClientParams params) { diff --git a/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/KafkaTestConfig.java b/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/KafkaTestConfig.java index 2770b648c..a06b9eea4 100644 --- a/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/KafkaTestConfig.java +++ b/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/config/KafkaTestConfig.java @@ -17,7 +17,6 @@ import org.apache.kafka.clients.producer.MockProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.boot.test.mock.mockito.MockReset; @@ -30,13 +29,10 @@ import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; import org.springframework.kafka.requestreply.RequestReplyFuture; import org.springframework.kafka.support.SendResult; import org.springframework.util.concurrent.ListenableFuture; -import ru.spcex.clearing.imdg.IMDGDistributedNames; -import ru.spcex.clearing.platform.messaging.service.RequestInfo; import ru.spcex.clearing.platform.messaging.service.sender.KafkaSender; import ru.spcex.clearing.test.TestUtils; -import ru.spcex.platform.imdg.api.Imdg; import ru.spcex.platform.imdg.api.ImdgId; -import ru.spcex.platform.imdg.api.ImdgProvider; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; @Configuration @Import(ImporterImdgTestConfig.class) @@ -132,9 +128,9 @@ public class KafkaTestConfig { @Bean("kafkaSender") public Supplier kafkaSender(@Qualifier("kafkaTestTemplate") KafkaTemplate kafkaTemplate, - @Qualifier("imdgProvider") ImdgProvider imdgProvider, + @Qualifier("hazelcastService") HazelcastService hazelcastService, Producer mockProducer) { - ImdgId imdgIdGenerator = imdgProvider.getImdgIdGenerator(); + ImdgId imdgIdGenerator = hazelcastService.getImdgIdGenerator(); return () -> KafkaSender .setup() .setKafkaTemplate(kafkaTemplate) diff --git a/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDBTest.java b/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDBTest.java index 34a191bbd..1987bb4b8 100644 --- a/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDBTest.java +++ b/clearing-parent/xml-importer/src/test/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDBTest.java @@ -59,7 +59,7 @@ class ImportToDBTest { private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); @Autowired - @Qualifier("imdgProvider") + @Qualifier("hazelcastService") private HazelcastService hazelcastService; @Autowired