From ed69bd7ec233a6b57522b2a894c405d875df107a Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Wed, 7 Aug 2024 16:39:47 +0300 Subject: [PATCH 1/8] notification sending on duplicate transaction number added in dbf-importer http://jira.mfd.msk:8088/browse/CLS-724 --- .../logic/data/tables/AbstractTable.java | 10 +++--- .../logic/data/tables/SDf57Table.java | 13 ++++--- .../logic/stages/DbfImportKafkaMessenger.java | 15 ++++---- .../dbf/importer/logic/stages/ImportToDB.java | 36 ++++++++++++------- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java index b5644ab41..a1c501ebc 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java @@ -1,13 +1,12 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables; +import java.math.BigDecimal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.spcex.platform.classes.base.SpcexObjectBase; import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; -import java.math.BigDecimal; - public abstract class AbstractTable { private final Logger log = LoggerFactory.getLogger(getClass()); private final String prefix; @@ -42,14 +41,13 @@ public abstract class AbstractTable { if (map == null) { bootMap(); } - if (checkOnExisting(obj)) { - map.insert(obj); - } + map.insert(obj); } - protected boolean checkOnExisting(T obj){ + public boolean checkOnExisting(T obj){ return true; } + protected void bootMap() { map = (ImdgHazelcast) hazelcastService.getImdg(nameOfMap, clazz); } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java index f2ce982fa..c7185b89e 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java @@ -1,15 +1,14 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ru.clearing.classes.statics.data.sdf.SDf57; -import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; -import ru.spcex.clearing.imdg.IMDGDistributedNames; - import java.math.BigDecimal; import java.time.Instant; import java.util.Collection; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ru.clearing.classes.statics.data.sdf.SDf57; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; public class SDf57Table extends AbstractTable { private final Logger log = LoggerFactory.getLogger(getClass()); @@ -83,7 +82,7 @@ public class SDf57Table extends AbstractTable { } @Override - protected boolean checkOnExisting(SDf57 obj) { + public boolean checkOnExisting(SDf57 obj) { if (map == null) { bootMap(); } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java index 3b6bf3287..597e99404 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java @@ -1,5 +1,11 @@ package ru.spcex.clearing.dbf.importer.logic.stages; +import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; @@ -19,13 +25,6 @@ import ru.spcex.platform.enumeration.ObjectType; import ru.spcex.platform.enumeration.Priority; import ru.spcex.platform.enumeration.SdfTable; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; - @Component public class DbfImportKafkaMessenger implements InitializingBean { final Logger log = LoggerFactory.getLogger(getClass()); @@ -82,7 +81,7 @@ public class DbfImportKafkaMessenger implements InitializingBean { } } - private void sendUserNotification(ObjectType objectType, String comment, Priority priority) { + public void sendUserNotification(ObjectType objectType, String comment, Priority priority) { final String destination = Consts.NOTIFICATION_NEW; NotificationNewRequest request = new NotificationNewRequest(); //request.setObjectId(); diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java index c92470d1b..22c53dc83 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java @@ -1,23 +1,26 @@ package ru.spcex.clearing.dbf.importer.logic.stages; -import com.linuxense.javadbf.DBFReader; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; -import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings; -import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; -import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; -import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult; -import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable; -import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; -import ru.spcex.platform.utils.log.ExceptionUtils; +import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01; +import com.linuxense.javadbf.DBFReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; import java.util.Map; - -import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import ru.clearing.classes.statics.data.sdf.SDf57; +import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings; +import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable; +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.enumeration.ObjectType; +import ru.spcex.platform.enumeration.Priority; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; +import ru.spcex.platform.utils.log.ExceptionUtils; /** * Заливка проверенных данных в базу @@ -58,7 +61,14 @@ public class ImportToDB extends Stage { log.warn("record index {} null", i); continue; } - table.injectEntity(table.getEntity(entity)); + SpcexObjectBase entityTable = table.getEntity(entity); + if (!table.checkOnExisting(entityTable) && entityTable instanceof SDf57) { + kafkaMessenger.sendUserNotification(ObjectType.rgst, + "Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее", + Priority.HIGH); + } else { + table.injectEntity(entityTable); + } } if (DF_01.equals(currTable)){ kafkaMessenger.sendPairSdfRequest(table, currTable.getPrefix()); From 6ccd8d3a3e3d5791110de58b773576eaea119dc9 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Wed, 7 Aug 2024 16:41:04 +0300 Subject: [PATCH 2/8] notification sending on duplicate transaction number added in xml-importer http://jira.mfd.msk:8088/browse/CLS-725 --- .../importer/config/ImporterImdgConfig.java | 2 +- .../data/tags/objects/DF01ObjectTag.java | 11 ++++--- .../data/tags/objects/DF04ObjectTag.java | 11 ++++--- .../data/tags/objects/DF06ObjectTag.java | 11 ++++--- .../data/tags/objects/DF52ObjectTag.java | 11 ++++--- .../data/tags/objects/DF55ObjectTag.java | 11 ++++--- .../data/tags/objects/DF57ObjectTag.java | 28 ++++++++++++++--- .../logic/data/tags/objects/ObjectTag.java | 31 ++++++++++++++----- .../xml/importer/logic/steps/ImportToDB.java | 29 ++++++++--------- .../logic/steps/XmlImportKafkaMessenger.java | 5 +-- .../importer/services/XMLImporterService.java | 6 +++- 11 files changed, 106 insertions(+), 50 deletions(-) diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java index 3789aef00..5a2800f81 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java @@ -34,7 +34,7 @@ public class ImporterImdgConfig { @Autowired @Bean - public ImdgProvider imdgProvider( + public HazelcastService imdgProvider( @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, ImportXMLServiceSettings settings diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java index a55a461c1..88797830f 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf01; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF01ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_01; + private static final Class CLAZZ = SDf01.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf01; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF01ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "CURR_CODE") private String currCode; @@ -50,10 +57,6 @@ public class DF01ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "FILE_TYPE") private String fileType; - public DF01ObjectTag() { - super(PREFIX); - } - @Override public SDf01 getSDfEntity() { SDf01 result = new SDf01(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java index b74b10e74..e40382446 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf04; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF04ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_04; + private static final Class CLAZZ = SDf04.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf04; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF04ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE") private String segType; @@ -61,10 +68,6 @@ public class DF04ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "IMP_RESULT") private String impResult; - public DF04ObjectTag() { - super(PREFIX); - } - @Override public SDf04 getSDfEntity() { SDf04 result = new SDf04(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java index 5da226924..334b0e653 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java @@ -8,12 +8,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf06; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF06ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_06; + private static final Class CLAZZ = SDf06.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf06; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF06ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT") private String account; @@ -54,10 +61,6 @@ public class DF06ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "PAY_VAL") private String payVal; - public DF06ObjectTag() { - super(PREFIX); - } - @Override public SDf06 getSDfEntity() { SDf06 result = new SDf06(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java index 45d1b3a20..7fc25418e 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf52; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF52ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_52; + private static final Class CLAZZ = SDf52.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf52; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF52ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT") private String account; @@ -32,10 +39,6 @@ public class DF52ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "ACC_TYPE") private String accType; - public DF52ObjectTag() { - super(PREFIX); - } - @Override public SDf52 getSDfEntity() { SDf52 result = new SDf52(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java index 508980cfc..ea7298862 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf55; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF55ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_55; + private static final Class CLAZZ = SDf55.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf55; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF55ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE") private String segType; @@ -123,10 +130,6 @@ public class DF55ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "SWIFT_INT") private String swiftInt; - public DF55ObjectTag() { - super(PREFIX); - } - @Override public SDf55 getSDfEntity() { SDf55 result = new SDf55(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java index 4df10aa9c..2baa0c917 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java @@ -5,14 +5,26 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.time.Instant; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Collection; +import java.util.Map; import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ru.clearing.classes.statics.data.sdf.SDf57; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF57ObjectTag extends ObjectTag { + private final Logger log = LoggerFactory.getLogger(getClass()); private static final ETable PREFIX = ETable.DF_57; + private static final Class CLAZZ = SDf57.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf57; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF57ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "ID") private Long id; @@ -116,10 +128,6 @@ public class DF57ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "KT_OUT") private String ktOut; - public DF57ObjectTag() { - super(PREFIX); - } - @Override public SDf57 getSDfEntity() { SDf57 result = new SDf57(); @@ -174,6 +182,18 @@ public class DF57ObjectTag extends ObjectTag { return result; } + @Override + public boolean checkOnExisting(SDf57 obj) { + if (map == null) { + bootMap(); + } + Collection sDf57s = map.getCollectionObjectsByFieldValues(Map.of("dbfId", obj.getDbfId())); + if (!sDf57s.isEmpty()) { + log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getDbfId()); + } + return sDf57s.isEmpty(); + } + public Long getId() { return id; } diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java index a746c5c82..533fecf8c 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; import ru.spcex.platform.classes.base.SpcexObjectBase; import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) @JsonSubTypes({ @@ -16,19 +17,25 @@ import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; @JsonSubTypes.Type(DF55ObjectTag.class), @JsonSubTypes.Type(DF57ObjectTag.class) }) -public abstract class ObjectTag { +public abstract class ObjectTag { private final ETable prefix; + private final Class clazz; + private final String nameOfMap; + protected HazelcastService hazelcastService; + protected ImdgHazelcast map; @JsonIgnore protected String fileName; @JsonIgnore protected Long generationId; - public ObjectTag(ETable prefix) { + public ObjectTag(ETable prefix, Class clazz, String nameOfMap) { this.prefix = prefix; + this.clazz = clazz; + this.nameOfMap = nameOfMap; } - public abstract T1 getSDfEntity(); + public abstract T getSDfEntity(); public ETable getPrefix() { return prefix; @@ -38,6 +45,10 @@ public abstract class ObjectTag { return fileName; } + public void setHazelcastService(HazelcastService hazelcastService) { + this.hazelcastService = hazelcastService; + } + public void setFileName(String fileName) { this.fileName = fileName; } @@ -50,14 +61,18 @@ public abstract class ObjectTag { this.generationId = generationId; } - public void insertEntity(ImdgHazelcast map) { - T1 obj = getSDfEntity(); - if (checkOnExisting(obj)) { - map.insert(obj); + public void injectEntity(T obj) { + if (map == null) { + bootMap(); } + map.insert(obj); } - protected boolean checkOnExisting(T1 obj) { + public boolean checkOnExisting(T obj) { return true; } + + protected void bootMap() { + map = (ImdgHazelcast) hazelcastService.getImdg(nameOfMap, clazz); + } } diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java index 35ce79a9f..a8ef05a82 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java @@ -1,36 +1,29 @@ package ru.spcex.clearing.xml.importer.logic.steps; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import ru.clearing.classes.statics.data.sdf.SDf57; import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult; import ru.spcex.clearing.xml.importer.logic.data.tags.DocumentTag; import ru.spcex.clearing.xml.importer.logic.data.tags.objects.ObjectTag; import ru.spcex.platform.classes.base.SpcexObjectBase; -import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; +import ru.spcex.platform.enumeration.ObjectType; +import ru.spcex.platform.enumeration.Priority; import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; @Component public class ImportToDB { private final Logger log = LoggerFactory.getLogger(getClass()); - private final XmlMapper xmlMapper; private final HazelcastService hazelcastService; private final XmlImportKafkaMessenger kafkaMessenger; - private final Map> mapOfTable; - public ImportToDB(@Qualifier("xmlMapper") XmlMapper xmlMapper, - HazelcastService hazelcastService, - XmlImportKafkaMessenger kafkaMessenger, - @Qualifier("mapOfTable") Map> mapOfTable) { - this.xmlMapper = xmlMapper; + public ImportToDB(HazelcastService hazelcastService, + XmlImportKafkaMessenger kafkaMessenger) { this.hazelcastService = hazelcastService; this.kafkaMessenger = kafkaMessenger; - this.mapOfTable = mapOfTable; } public StageResult process(ResultContainer resultContainer) { @@ -41,10 +34,18 @@ public class ImportToDB { String fileName = resultContainer.getXmlFile().getName(); Long fileId = hazelcastService.getImdgIdGenerator().nextId(); for (ObjectTag o : documentTag.getObjects()) { - ImdgHazelcast imdgMap = mapOfTable.get(o.getPrefix()); + o.setHazelcastService(hazelcastService); o.setFileName(fileName); o.setGenerationId(fileId); - o.insertEntity(imdgMap); + + SpcexObjectBase entityTable = o.getSDfEntity(); + if (!o.checkOnExisting(entityTable) && entityTable instanceof SDf57) { + kafkaMessenger.sendUserNotification(ObjectType.rgst, + "Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее", + Priority.HIGH); + } else { + o.injectEntity(entityTable); + } } if (ETable.DF_01.equals(currTable)) { diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java index 15c1f3bf0..166fcdb8f 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java @@ -1,5 +1,7 @@ package ru.spcex.clearing.xml.importer.logic.steps; +import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; + import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; @@ -9,7 +11,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.stereotype.Component; import ru.spcex.clearing.platform.messaging.domain.Consts; -import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; import ru.spcex.clearing.platform.messaging.domain.cud.balance.StatementRequest; import ru.spcex.clearing.platform.messaging.domain.cud.clearing.Sdf04Request; import ru.spcex.clearing.platform.messaging.domain.cud.utilities.NotificationNewRequest; @@ -76,7 +77,7 @@ public class XmlImportKafkaMessenger implements InitializingBean { } } - private void sendUserNotification(ObjectType objectType, String comment, Priority priority) { + public void sendUserNotification(ObjectType objectType, String comment, Priority priority) { final String destination = Consts.NOTIFICATION_NEW; NotificationNewRequest request = new NotificationNewRequest(); request.setObjectType(objectType.getKey()); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java index b803e8a70..4e0f4d602 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java @@ -59,7 +59,11 @@ public class XMLImporterService { } public void processTable(ETable specificTable) { - log.info("adding import task {}", specificTable == null ? "without specific table" : specificTable); + if (specificTable == null) { + log.trace("adding import task without specific table"); + } else { + log.info("adding import task {}", specificTable); + } fileChecker.checkAndLoadSFTP(); executorService.execute(() -> { From 27c2510588237259228bfa318e564eb1b479df72 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Wed, 7 Aug 2024 16:41:27 +0300 Subject: [PATCH 3/8] notification sending on duplicate transaction number added in swt-importer http://jira.mfd.msk:8088/browse/CLS-726 --- .../logic/data/tables/AbstractTable.java | 8 ++++ .../logic/data/tables/SDf20Table.java | 22 ++++++++-- .../logic/data/tables/SDf21Table.java | 28 ++++--------- .../data/tables/WithTransactionNumber.java | 4 ++ .../swt/importer/logic/stages/ImportToDB.java | 42 ++++++++++++------- .../logic/stages/SWTImportKafkaMessenger.java | 2 +- 6 files changed, 66 insertions(+), 40 deletions(-) create mode 100644 clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java index e675ac155..07d896ef5 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java @@ -34,6 +34,10 @@ public abstract class AbstractTable { public abstract T getEntity(SWTRecord record); + public String getPrefix() { + return prefix; + } + public void injectEntity(T obj) { if (map == null) { bootMap(); @@ -41,6 +45,10 @@ public abstract class AbstractTable { map.insert(obj); } + public boolean checkOnExisting(T obj){ + return true; + } + protected void bootMap() { map = (ImdgHazelcast) hazelcastService.getImdg(nameOfMap, clazz); } diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java index e31825c51..e0914d527 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java @@ -1,14 +1,17 @@ package ru.spcex.clearing.swt.importer.logic.data.tables; +import java.time.Instant; +import java.util.Collection; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ru.clearing.classes.statics.data.sdf.SDf20; import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; import ru.spcex.clearing.swt.importer.readers.SWTRecord; -import java.time.Instant; - -public class SDf20Table extends AbstractTable { - +public class SDf20Table extends AbstractTable implements WithTransactionNumber { + private final Logger log = LoggerFactory.getLogger(getClass()); private static final String PREFIX = ETable.S_DF_20.name(); private static final Class CLAZZ = SDf20.class; private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf20; @@ -41,4 +44,15 @@ public class SDf20Table extends AbstractTable { return result; } + @Override + public boolean checkOnExisting(SDf20 obj) { + if (map == null) { + bootMap(); + } + Collection sDf20s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber())); + if (!sDf20s.isEmpty()) { + log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber()); + } + return sDf20s.isEmpty(); + } } \ No newline at end of file diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java index cb8038b50..076868aa6 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java @@ -1,19 +1,17 @@ package ru.spcex.clearing.swt.importer.logic.data.tables; +import java.time.Instant; +import java.util.Collection; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.clearing.classes.statics.data.sdf.SDf21; import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; import ru.spcex.clearing.swt.importer.readers.SWTRecord; -import ru.spcex.platform.imdg.api.predicate.ImdgPredicate; -import ru.spcex.platform.imdg.api.predicate.ImdgPredicateBuilder; -import java.time.Instant; - -public class SDf21Table extends AbstractTable { +public class SDf21Table extends AbstractTable implements WithTransactionNumber { private final Logger log = LoggerFactory.getLogger(getClass()); - private static final String PREFIX = ETable.S_DF_21.name(); private static final Class CLAZZ = SDf21.class; private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf21; @@ -46,22 +44,14 @@ public class SDf21Table extends AbstractTable { } @Override - public void injectEntity(SDf21 obj) { + public boolean checkOnExisting(SDf21 obj) { if (map == null) { bootMap(); } - ImdgPredicateBuilder pb = map.predicateBuilder(); - ImdgPredicate sameTransactionNumberAndOperationCode = pb.and( - pb.equals("transactionNumber", obj.getTransactionNumber()), - pb.equals("operationCode", obj.getOperationCode()) - ); - SDf21 firstObjectBySQL = map.getFirstObjectByPredicate( - sameTransactionNumberAndOperationCode - ); - if (firstObjectBySQL != null) { - log.warn("Skip insert by {}, sdf21.transactionNumber: {}", PREFIX, obj.getTransactionNumber()); - } else { - map.insert(obj); + Collection sDf21s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber())); + if (!sDf21s.isEmpty()) { + log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber()); } + return sDf21s.isEmpty(); } } \ No newline at end of file diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java new file mode 100644 index 000000000..42a6d4993 --- /dev/null +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java @@ -0,0 +1,4 @@ +package ru.spcex.clearing.swt.importer.logic.data.tables; + +public interface WithTransactionNumber { +} diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java index 9895d7fb2..aadb9dee0 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java @@ -1,17 +1,6 @@ package ru.spcex.clearing.swt.importer.logic.stages; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; -import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings; -import ru.spcex.clearing.swt.importer.logic.data.ResultContainer; -import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; -import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult; -import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable; -import ru.spcex.clearing.swt.importer.readers.SWTReader; -import ru.spcex.clearing.swt.importer.readers.SWTRecord; -import ru.spcex.clearing.swt.importer.readers.ValidationResult; -import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; -import ru.spcex.platform.utils.log.ExceptionUtils; +import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -20,8 +9,22 @@ import java.nio.charset.Charset; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; - -import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings; +import ru.spcex.clearing.swt.importer.logic.data.ResultContainer; +import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; +import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult; +import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable; +import ru.spcex.clearing.swt.importer.logic.data.tables.WithTransactionNumber; +import ru.spcex.clearing.swt.importer.readers.SWTReader; +import ru.spcex.clearing.swt.importer.readers.SWTRecord; +import ru.spcex.clearing.swt.importer.readers.ValidationResult; +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.enumeration.ObjectType; +import ru.spcex.platform.enumeration.Priority; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; +import ru.spcex.platform.utils.log.ExceptionUtils; /** * Заливка проверенных данных в базу @@ -68,8 +71,15 @@ public class ImportToDB extends Stage { List records = swtReader.getGroupRecords(); records.forEach(record -> { if (record.isNotEmpty()) { - table.injectEntity(table.getEntity(record)); - log.debug("{} entity stored.", counter.incrementAndGet()); + SpcexObjectBase entityTable = table.getEntity(record); + if (!table.checkOnExisting(entityTable) && table instanceof WithTransactionNumber) { + kafkaMessenger.sendUserNotification(ObjectType.rgst, + "Номер транзакции " + record.getValFor23Tag() + " в полученном " + table.getPrefix() + " уже был обработан ранее", + Priority.HIGH); + } else { + table.injectEntity(entityTable); + log.debug("{} entity stored.", counter.incrementAndGet()); + } } }); } while (swtReader.hasNextRecord()); diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java index 952469534..5433f3915 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java @@ -84,7 +84,7 @@ public class SWTImportKafkaMessenger implements InitializingBean { // } } - private void sendUserNotification(ObjectType objectType, String comment, Priority priority) { + public void sendUserNotification(ObjectType objectType, String comment, Priority priority) { final String destination = Consts.NOTIFICATION_NEW; NotificationNewRequest request = new NotificationNewRequest(); //request.setObjectId(); From 4994ead0d8df53b2a8473d5606e1ea0300232cfa Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Wed, 7 Aug 2024 16:41:46 +0300 Subject: [PATCH 4/8] pom fixed --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index e9f73aa7b..58c235554 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,8 @@ ${folder_root_clearing}/clearing-parent/registry-service ${folder_root_clearing}/clearing-parent/scheduler-service ${folder_root_clearing}/clearing-parent/gateway-api + ${folder_root_clearing}/clearing-parent/xml-exporter + ${folder_root_clearing}/clearing-parent/xml-importer 3.12.4 1.7.33 From 8d70db939f81e0472cd4363d110d84eff3a44a7f Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Wed, 7 Aug 2024 16:59:24 +0300 Subject: [PATCH 5/8] added allowed branched to .gitlab-ci.yml --- .gitlab-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4cbe55e15..d547d8169 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,8 @@ preflight: expire_in: 1 day paths: - clearing_test + only: + - dev build: image: ${DOCKER_PROXY}/maven:latest @@ -34,6 +36,8 @@ build: expire_in: 1 day paths: - /builds/mfd/clearing/z-distr/target/clearing + only: + - dev test: stage: test @@ -50,4 +54,6 @@ test: expire_in: 1 day when: always paths: - - $CI_PROJECT_DIR/clr_test/mnt \ No newline at end of file + - $CI_PROJECT_DIR/clr_test/mnt + only: + - dev \ No newline at end of file From fe1279ae63ef70dea335dde9bd113c28666efe0b Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 8 Aug 2024 11:04:12 +0300 Subject: [PATCH 6/8] SDf20Table duplicate code fixed --- .../clearing/swt/importer/logic/data/tables/SDf20Table.java | 1 - 1 file changed, 1 deletion(-) diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java index e0914d527..559919806 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java @@ -40,7 +40,6 @@ public class SDf20Table extends AbstractTable implements WithTransactionN result.setSecurityName(record.getValFor35BTag()[1]); result.setSecurityType(record.getValFor35BTag()[2]); result.setNameCl(record.getValFor83DTag()[1]); - result.setTransactionNumber(record.getValFor23Tag()); return result; } From 43162aa18a62e6d86b331f5d34a0cc9e73052c3f Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 8 Aug 2024 11:51:29 +0300 Subject: [PATCH 7/8] xml-importer and xml-exporter added to z-distr --- pom.xml | 2 + z-distr/pom.xml | 46 +++++++++++++++++++ .../src/main/resources/distr/sh/kill_all.sh | 4 +- .../src/main/resources/distr/sh/launch_all.sh | 2 + .../main/resources/distr/sh/xml-exporter.sh | 9 ++++ .../main/resources/distr/sh/xml-importer.sh | 9 ++++ 6 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 z-distr/src/main/resources/distr/sh/xml-exporter.sh create mode 100644 z-distr/src/main/resources/distr/sh/xml-importer.sh diff --git a/pom.xml b/pom.xml index e9f73aa7b..58c235554 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,8 @@ ${folder_root_clearing}/clearing-parent/registry-service ${folder_root_clearing}/clearing-parent/scheduler-service ${folder_root_clearing}/clearing-parent/gateway-api + ${folder_root_clearing}/clearing-parent/xml-exporter + ${folder_root_clearing}/clearing-parent/xml-importer 3.12.4 1.7.33 diff --git a/z-distr/pom.xml b/z-distr/pom.xml index c168e620a..9af0f0e1f 100644 --- a/z-distr/pom.xml +++ b/z-distr/pom.xml @@ -568,6 +568,52 @@ + + copy-xml-exporter-bin + prepare-package + + copy + + + + + ${folder_root_xml-exporter}/target/xml-exporter.jar + ${folder.clearing.distr.bin}/xml-exporter.jar + + + ${folder_root_xml-exporter}/target/xml-exporter.jar + ${folder.clearing.distr.services}/xml-exporter/xml-exporter.jar + + + ${folder_root_xml-exporter}/src/main/resources/application.properties + ${folder.clearing.distr.settings}/xml-exporter/application.properties + + + + + + copy-xml-importer-bin + prepare-package + + copy + + + + + ${folder_root_xml-importer}/target/xml-importer.jar + ${folder.clearing.distr.bin}/xml-importer.jar + + + ${folder_root_xml-importer}/target/xml-importer.jar + ${folder.clearing.distr.services}/xml-importer/xml-importer.jar + + + ${folder_root_xml-importer}/src/main/resources/application.properties + ${folder.clearing.distr.settings}/xml-importer/application.properties + + + + diff --git a/z-distr/src/main/resources/distr/sh/kill_all.sh b/z-distr/src/main/resources/distr/sh/kill_all.sh index 46da22a76..8e9a2a8a5 100644 --- a/z-distr/src/main/resources/distr/sh/kill_all.sh +++ b/z-distr/src/main/resources/distr/sh/kill_all.sh @@ -19,4 +19,6 @@ kill -9 $(ps -ef | grep java | grep scheduler-service.jar | awk '{print $2}') kill -9 $(ps -ef | grep java | grep reports-service.jar | awk '{print $2}') kill -9 $(ps -ef | grep java | grep registry-service.jar | awk '{print $2}') kill -9 $(ps -ef | grep java | grep swt-importer.jar | awk '{print $2}') -kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}') \ No newline at end of file +kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}') +kill -9 $(ps -ef | grep java | grep xml-exporter.jar | awk '{print $2}') +kill -9 $(ps -ef | grep java | grep xml-importer.jar | awk '{print $2}') \ No newline at end of file diff --git a/z-distr/src/main/resources/distr/sh/launch_all.sh b/z-distr/src/main/resources/distr/sh/launch_all.sh index 7447e407d..b4c6a45be 100644 --- a/z-distr/src/main/resources/distr/sh/launch_all.sh +++ b/z-distr/src/main/resources/distr/sh/launch_all.sh @@ -20,3 +20,5 @@ cd /opt/mfd/clearing/bin /opt/mfd/clearing/bin/registry-service.sh /opt/mfd/clearing/bin/swt-importer.sh /opt/mfd/clearing/bin/gateway-api.sh +/opt/mfd/clearing/bin/xml-exporter.sh +/opt/mfd/clearing/bin/xml-importer.sh diff --git a/z-distr/src/main/resources/distr/sh/xml-exporter.sh b/z-distr/src/main/resources/distr/sh/xml-exporter.sh new file mode 100644 index 000000000..862968408 --- /dev/null +++ b/z-distr/src/main/resources/distr/sh/xml-exporter.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +CLEARING_HOME=/opt/mfd/clearing/ +cd $CLEARING_HOME/bin + +CMD="java -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=7031 -jar xml-exporter.jar --spring.config.location=$CLEARING_HOME/settings/xml-exporter/" + +$CMD >/dev/null 2>&1 & + diff --git a/z-distr/src/main/resources/distr/sh/xml-importer.sh b/z-distr/src/main/resources/distr/sh/xml-importer.sh new file mode 100644 index 000000000..d4f0d9ec7 --- /dev/null +++ b/z-distr/src/main/resources/distr/sh/xml-importer.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +CLEARING_HOME=/opt/mfd/clearing/ +cd $CLEARING_HOME/bin + +CMD="java -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=7041 -jar xml-importer.jar --spring.config.location=$CLEARING_HOME/settings/xml-importer/" + +$CMD >/dev/null 2>&1 & + From 20b1eef52d020e697c02d3378f2aef50596dd5eb Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 8 Aug 2024 13:34:54 +0300 Subject: [PATCH 8/8] resource_group added to .gitlab-ci.yml to control concurrent tests --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d547d8169..c4b92b55b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ preflight: - clearing_test only: - dev + resource_group: clearing-test build: image: ${DOCKER_PROXY}/maven:latest @@ -38,6 +39,7 @@ build: - /builds/mfd/clearing/z-distr/target/clearing only: - dev + resource_group: clearing-test test: stage: test @@ -56,4 +58,5 @@ test: paths: - $CI_PROJECT_DIR/clr_test/mnt only: - - dev \ No newline at end of file + - dev + resource_group: clearing-test \ No newline at end of file