From 54722afb2c4ba36683e60d7dec9dbaf485e6d826 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 24 May 2024 18:58:19 +0300 Subject: [PATCH] Pipelines removed, minor code changes in ObjectTags --- .../importer/config/XMLImporterConfig.java | 16 ------- .../xml/importer/logic/Processor.java | 43 ------------------- .../data/tags/objects/DF04ObjectTag.java | 16 +++---- .../data/tags/objects/DF55ObjectTag.java | 28 ++++++------ .../data/tags/objects/DF57ObjectTag.java | 28 ++++++------ .../xml/importer/logic/stages/Stage.java | 16 ------- .../ChangeDirOfFileStage.java | 13 +++--- .../logic/{stages => steps}/ImportToDB.java | 8 ++-- .../XmlImportKafkaMessenger.java | 2 +- .../importer/services/XMLImporterService.java | 31 ++++++++++--- 10 files changed, 73 insertions(+), 128 deletions(-) delete mode 100644 clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/Processor.java delete mode 100644 clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/Stage.java rename clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/{stages => steps}/ChangeDirOfFileStage.java (92%) rename clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/{stages => steps}/ImportToDB.java (93%) rename clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/{stages => steps}/XmlImportKafkaMessenger.java (99%) diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java index 84336310d..8111a6ec0 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java @@ -8,8 +8,6 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; @@ -30,9 +28,6 @@ import ru.spcex.clearing.xml.importer.logic.data.tags.objects.DF52ObjectTag; import ru.spcex.clearing.xml.importer.logic.data.tags.objects.DF55ObjectTag; import ru.spcex.clearing.xml.importer.logic.data.tags.objects.DF57ObjectTag; import ru.spcex.clearing.xml.importer.logic.data.tags.objects.ObjectTag; -import ru.spcex.clearing.xml.importer.logic.stages.ChangeDirOfFileStage; -import ru.spcex.clearing.xml.importer.logic.stages.ImportToDB; -import ru.spcex.clearing.xml.importer.logic.stages.Stage; import ru.spcex.platform.classes.base.SpcexObjectBase; @Configuration @@ -48,17 +43,6 @@ public class XMLImporterConfig { this.context = context; } - @Bean("pipeline") - public List pipeline() { - List pipeline = new LinkedList<>(); - log.info("XMLImporterConfig.pipeline"); - - pipeline.add(context.getBean(ImportToDB.class)); - pipeline.add(context.getBean(ChangeDirOfFileStage.class)); - - return pipeline; - } - @Bean("executor") public ThreadPoolTaskExecutor executor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/Processor.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/Processor.java deleted file mode 100644 index 601e7be28..000000000 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/Processor.java +++ /dev/null @@ -1,43 +0,0 @@ -package ru.spcex.clearing.xml.importer.logic; - -import java.util.Arrays; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; -import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; -import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult; -import ru.spcex.clearing.xml.importer.logic.stages.Stage; - -@Component("processor") -public class Processor { - private final Logger log = LoggerFactory.getLogger(getClass()); - private final List pipeline; - - public Processor(@Qualifier("pipeline") List pipeline) { - this.pipeline = pipeline; - } - - public void process(ResultContainer task) { - log.info("uuid {}. Task started", task.getUuid()); - long startMills = System.currentTimeMillis(); - for (Stage currStage : pipeline) { - if (statusIsFinal(task.getLastStageStatus()) && currStage.skipCompleted()) { - continue; - } - log.info("uuid {}. Stage: {}", task.getUuid(), currStage.getClass().getSimpleName()); - task.setLastStageStatus(currStage.process(task)); - log.info("uuid {}. Stage {} finished with status {}", task.getUuid(), currStage.getClass().getSimpleName(), task.getLastStageStatus()); - } - long endMills = System.currentTimeMillis(); - log.info("uuid {}. Task completed, result: {}, time working: {} ms", - task.getUuid(), - task, - endMills - startMills); - } - - private boolean statusIsFinal(StageResult previousStageStatus) { - return Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(previousStageStatus); - } -} diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java index 01f123241..05a668be0 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java @@ -76,16 +76,16 @@ public class DF04ObjectTag extends ObjectTag { result.setDocnmprev(entity.getDocnmprev()); result.setC_acc_deb(entity.getcAccDeb()); result.setSbanknam1(entity.getSbanknam1()); - result.setSbanknam2(entity.getSbanknam2()); - result.setSbanknam3(entity.getSbanknam3()); - result.setSbanknam4(entity.getSbanknam4()); - result.setSbanknam5(entity.getSbanknam5()); + result.setSbanknam2(entity.getSbanknam1()); + result.setSbanknam3(entity.getSbanknam1()); + result.setSbanknam4(entity.getSbanknam1()); + result.setSbanknam5(entity.getSbanknam1()); result.setC_acc_cred(entity.getcAccCred()); result.setRbanknam1(entity.getRbanknam1()); - result.setRbanknam2(entity.getRbanknam2()); - result.setRbanknam3(entity.getRbanknam3()); - result.setRbanknam4(entity.getRbanknam4()); - result.setRbanknam5(entity.getRbanknam5()); + result.setRbanknam2(entity.getRbanknam1()); + result.setRbanknam3(entity.getRbanknam1()); + result.setRbanknam4(entity.getRbanknam1()); + result.setRbanknam5(entity.getRbanknam1()); result.setPay_date(entity.getPayDate().toString()); result.setPay_val(entity.getPayVal()); result.setSum_deb(entity.getSumDeb()); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java index f5decccff..69e40aad1 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java @@ -139,33 +139,33 @@ public class DF55ObjectTag extends ObjectTag { result.setSbankcode(entity.getSbankcode()); result.setC_acc_deb(entity.getcAccDeb()); result.setSbanknam1(entity.getSbanknam1()); - result.setSbanknam2(entity.getSbanknam2()); - result.setSbanknam3(entity.getSbanknam3()); - result.setSbanknam4(entity.getSbanknam4()); - result.setSbanknam5(entity.getSbanknam5()); + result.setSbanknam2(entity.getSbanknam1()); + result.setSbanknam3(entity.getSbanknam1()); + result.setSbanknam4(entity.getSbanknam1()); + result.setSbanknam5(entity.getSbanknam1()); result.setRbankcode(entity.getRbankcode()); result.setC_acc_cred(entity.getcAccCred()); result.setRbanknam1(entity.getRbanknam1()); - result.setRbanknam2(entity.getRbanknam2()); - result.setRbanknam3(entity.getRbanknam3()); - result.setRbanknam4(entity.getRbanknam4()); - result.setRbanknam5(entity.getRbanknam5()); + result.setRbanknam2(entity.getRbanknam1()); + result.setRbanknam3(entity.getRbanknam1()); + result.setRbanknam4(entity.getRbanknam1()); + result.setRbanknam5(entity.getRbanknam1()); result.setOp_type(entity.getOpType()); result.setOp_order(entity.getOpOrder()); result.setPay_date(entity.getPayDate().toString()); result.setPay_val(entity.getPayVal()); result.setSum_deb(entity.getSumDeb()); result.setSclientn1(entity.getSclientn1()); - result.setSclientn2(entity.getSclientn2()); - result.setSclientn3(entity.getSclientn3()); - result.setSclientn4(entity.getSclientn4()); + result.setSclientn2(entity.getSclientn1()); + result.setSclientn3(entity.getSclientn1()); + result.setSclientn4(entity.getSclientn1()); result.setInn_deb(entity.getInnCred()); result.setKpp_deb(entity.getKppCred()); result.setAcc_deb(entity.getAccDeb()); result.setRclientn1(entity.getRclientn1()); - result.setRclientn2(entity.getRclientn2()); - result.setRclientn3(entity.getRclientn3()); - result.setRclientn4(entity.getRclientn4()); + result.setRclientn2(entity.getRclientn1()); + result.setRclientn3(entity.getRclientn1()); + result.setRclientn4(entity.getRclientn1()); result.setInn_cred(entity.getInnCred()); result.setKpp_cred(entity.getKppCred()); result.setAcc_kr_1(entity.getAccKr1()); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java index 670bce7c4..e9738772b 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java @@ -131,33 +131,33 @@ public class DF57ObjectTag extends ObjectTag { result.setSbankcode(entity.getSbankcode()); result.setC_acc_deb(entity.getcAccDeb()); result.setSbanknam1(entity.getSbanknam1()); - result.setSbanknam2(entity.getSbanknam2()); - result.setSbanknam3(entity.getSbanknam3()); - result.setSbanknam4(entity.getSbanknam4()); - result.setSbanknam5(entity.getSbanknam5()); + result.setSbanknam2(entity.getSbanknam1()); + result.setSbanknam3(entity.getSbanknam1()); + result.setSbanknam4(entity.getSbanknam1()); + result.setSbanknam5(entity.getSbanknam1()); result.setRbankcode(entity.getRbankcode()); result.setC_acc_cred(entity.getcAccCred()); result.setRbanknam1(entity.getRbanknam1()); - result.setRbanknam2(entity.getRbanknam2()); - result.setRbanknam3(entity.getRbanknam3()); - result.setRbanknam4(entity.getRbanknam4()); - result.setRbanknam5(entity.getRbanknam5()); + result.setRbanknam2(entity.getRbanknam1()); + result.setRbanknam3(entity.getRbanknam1()); + result.setRbanknam4(entity.getRbanknam1()); + result.setRbanknam5(entity.getRbanknam1()); result.setOp_type(entity.getOpType()); result.setPay_date(entity.getPayDate().toString()); result.setExt_date(entity.getExtDate().toString()); result.setPay_val(entity.getPayVal()); result.setSum_deb(entity.getSumDeb()); result.setSclientn1(entity.getSclientn1()); - result.setSclientn2(entity.getSclientn2()); - result.setSclientn3(entity.getSclientn3()); - result.setSclientn4(entity.getSclientn4()); + result.setSclientn2(entity.getSclientn1()); + result.setSclientn3(entity.getSclientn1()); + result.setSclientn4(entity.getSclientn1()); result.setInn_deb(entity.getInnDeb()); result.setKpp_deb(entity.getKppDeb()); result.setAcc_deb(entity.getAccDeb()); result.setRclientn1(entity.getRclientn1()); - result.setRclientn2(entity.getRclientn2()); - result.setRclientn3(entity.getRclientn3()); - result.setRclientn4(entity.getRclientn4()); + result.setRclientn2(entity.getRclientn1()); + result.setRclientn3(entity.getRclientn1()); + result.setRclientn4(entity.getRclientn1()); result.setInn_cred(entity.getInnDeb()); result.setKpp_cred(entity.getKppDeb()); result.setAcc_kr(entity.getAccKr()); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/Stage.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/Stage.java deleted file mode 100644 index cec98f8f6..000000000 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/Stage.java +++ /dev/null @@ -1,16 +0,0 @@ -package ru.spcex.clearing.xml.importer.logic.stages; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; -import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult; - -public abstract class Stage { - protected final Logger log = LoggerFactory.getLogger(getClass()); - - public abstract StageResult process(ResultContainer resultContainer); - - public boolean skipCompleted() { - return true; - } -} diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/ChangeDirOfFileStage.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ChangeDirOfFileStage.java similarity index 92% rename from clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/ChangeDirOfFileStage.java rename to clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ChangeDirOfFileStage.java index 7e24d9bbf..0f1f1db31 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/ChangeDirOfFileStage.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ChangeDirOfFileStage.java @@ -1,4 +1,4 @@ -package ru.spcex.clearing.xml.importer.logic.stages; +package ru.spcex.clearing.xml.importer.logic.steps; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -7,6 +7,8 @@ import java.io.IOException; import java.nio.file.Files; import java.time.Instant; import java.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import ru.spcex.clearing.xml.importer.config.settings.ImportXMLServiceSettings; import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; @@ -14,7 +16,8 @@ import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult; import ru.spcex.platform.utils.time.TimeUtil; @Component -public class ChangeDirOfFileStage extends Stage { +public class ChangeDirOfFileStage { + private final Logger log = LoggerFactory.getLogger(getClass()); private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy.MM.dd HH.mm.ss"); private final ImportXMLServiceSettings settings; @@ -22,12 +25,6 @@ public class ChangeDirOfFileStage extends Stage { this.settings = settings; } - @Override - public boolean skipCompleted() { - return false; - } - - @Override public StageResult process(ResultContainer resultContainer) { File srcDir = new File(settings.getStore().getSrcDir()); File outDir; diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/ImportToDB.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java similarity index 93% rename from clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/ImportToDB.java rename to clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java index f83718f7d..d945e0ce4 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/ImportToDB.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java @@ -1,9 +1,11 @@ -package ru.spcex.clearing.xml.importer.logic.stages; +package ru.spcex.clearing.xml.importer.logic.steps; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import java.io.File; import java.io.IOException; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; @@ -16,7 +18,8 @@ import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; import ru.spcex.platform.utils.log.ExceptionUtils; @Component -public class ImportToDB extends Stage { +public class ImportToDB { + private final Logger log = LoggerFactory.getLogger(getClass()); private final XmlMapper xmlMapper; private final HazelcastService hazelcastService; private final Map> objectTagMap; @@ -32,7 +35,6 @@ public class ImportToDB extends Stage { this.kafkaMessenger = kafkaMessenger; } - @Override public StageResult process(ResultContainer resultContainer) { ETable currTable = resultContainer.getXmlTable(); File xmlFile = resultContainer.getXmlFile(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/XmlImportKafkaMessenger.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java similarity index 99% rename from clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/XmlImportKafkaMessenger.java rename to clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java index 1a405ba19..c64c82982 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/stages/XmlImportKafkaMessenger.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java @@ -1,4 +1,4 @@ -package ru.spcex.clearing.xml.importer.logic.stages; +package ru.spcex.clearing.xml.importer.logic.steps; import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java index d29a6fc60..76cad50da 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java @@ -20,9 +20,10 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; -import ru.spcex.clearing.xml.importer.logic.Processor; import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; +import ru.spcex.clearing.xml.importer.logic.steps.ChangeDirOfFileStage; +import ru.spcex.clearing.xml.importer.logic.steps.ImportToDB; import ru.spcex.platform.utils.collection.Pair; @Service("xmlImporterService") @@ -31,15 +32,19 @@ public class XMLImporterService { private final Logger log = LoggerFactory.getLogger(getClass()); private final FileChecker fileChecker; private final ThreadPoolTaskExecutor executorService; - private final Processor processor; private final Set filesCurrentlyInProcess; + private final ImportToDB importToDB; + private final ChangeDirOfFileStage changeDirOfFileStage; + public XMLImporterService(@Qualifier("fileChecker") FileChecker fileChecker, @Qualifier("executor") ThreadPoolTaskExecutor executorService, - @Qualifier("processor") Processor processor) { + ImportToDB importToDB, + ChangeDirOfFileStage changeDirOfFileStage) { this.fileChecker = fileChecker; this.executorService = executorService; - this.processor = processor; + this.importToDB = importToDB; + this.changeDirOfFileStage = changeDirOfFileStage; this.filesCurrentlyInProcess = new HashSet<>(); } @@ -82,7 +87,23 @@ public class XMLImporterService { for (Pair newFilesEntry : orderByTimeFiles) { ETable currTable = newFilesEntry.getSecond(); File xmlFile = newFilesEntry.getFirst(); - processor.process(ResultContainer.createNewTask(currTable, xmlFile)); + ResultContainer task = ResultContainer.createNewTask(currTable, xmlFile); + + log.info("uuid {}. Task started", task.getUuid()); + long startMills = System.currentTimeMillis(); + + log.info("uuid {}. Stage: Import to DB", task.getUuid()); + task.setLastStageStatus(importToDB.process(task)); + log.info("uuid {}. Stage import to DB finished with status {}", task.getUuid(), task.getLastStageStatus()); + log.info("uuid {}. Stage: Change directory of file", task.getUuid()); + task.setLastStageStatus(changeDirOfFileStage.process(task)); + log.info("uuid {}. Stage change directory of file finished with status {}", task.getUuid(), task.getLastStageStatus()); + + long endMills = System.currentTimeMillis(); + log.info("uuid {}. Task completed, result: {}, time working: {} ms", + task.getUuid(), + task, + endMills - startMills); } } finally { if (newFiles != null && !newFiles.isEmpty()) {