diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java index b5644ab41..a1c501ebc 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java @@ -1,13 +1,12 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables; +import java.math.BigDecimal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.spcex.platform.classes.base.SpcexObjectBase; import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; -import java.math.BigDecimal; - public abstract class AbstractTable { private final Logger log = LoggerFactory.getLogger(getClass()); private final String prefix; @@ -42,14 +41,13 @@ public abstract class AbstractTable { if (map == null) { bootMap(); } - if (checkOnExisting(obj)) { - map.insert(obj); - } + map.insert(obj); } - protected boolean checkOnExisting(T obj){ + public boolean checkOnExisting(T obj){ return true; } + protected void bootMap() { map = (ImdgHazelcast) hazelcastService.getImdg(nameOfMap, clazz); } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java index f2ce982fa..c7185b89e 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java @@ -1,15 +1,14 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ru.clearing.classes.statics.data.sdf.SDf57; -import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; -import ru.spcex.clearing.imdg.IMDGDistributedNames; - import java.math.BigDecimal; import java.time.Instant; import java.util.Collection; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ru.clearing.classes.statics.data.sdf.SDf57; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; public class SDf57Table extends AbstractTable { private final Logger log = LoggerFactory.getLogger(getClass()); @@ -83,7 +82,7 @@ public class SDf57Table extends AbstractTable { } @Override - protected boolean checkOnExisting(SDf57 obj) { + public boolean checkOnExisting(SDf57 obj) { if (map == null) { bootMap(); } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java index 3b6bf3287..597e99404 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java @@ -1,5 +1,11 @@ package ru.spcex.clearing.dbf.importer.logic.stages; +import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; @@ -19,13 +25,6 @@ import ru.spcex.platform.enumeration.ObjectType; import ru.spcex.platform.enumeration.Priority; import ru.spcex.platform.enumeration.SdfTable; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; - @Component public class DbfImportKafkaMessenger implements InitializingBean { final Logger log = LoggerFactory.getLogger(getClass()); @@ -82,7 +81,7 @@ public class DbfImportKafkaMessenger implements InitializingBean { } } - private void sendUserNotification(ObjectType objectType, String comment, Priority priority) { + public void sendUserNotification(ObjectType objectType, String comment, Priority priority) { final String destination = Consts.NOTIFICATION_NEW; NotificationNewRequest request = new NotificationNewRequest(); //request.setObjectId(); diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java index c92470d1b..22c53dc83 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java @@ -1,23 +1,26 @@ package ru.spcex.clearing.dbf.importer.logic.stages; -import com.linuxense.javadbf.DBFReader; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; -import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings; -import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; -import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; -import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult; -import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable; -import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; -import ru.spcex.platform.utils.log.ExceptionUtils; +import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01; +import com.linuxense.javadbf.DBFReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; import java.util.Map; - -import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import ru.clearing.classes.statics.data.sdf.SDf57; +import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings; +import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable; +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.enumeration.ObjectType; +import ru.spcex.platform.enumeration.Priority; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; +import ru.spcex.platform.utils.log.ExceptionUtils; /** * Заливка проверенных данных в базу @@ -58,7 +61,14 @@ public class ImportToDB extends Stage { log.warn("record index {} null", i); continue; } - table.injectEntity(table.getEntity(entity)); + SpcexObjectBase entityTable = table.getEntity(entity); + if (!table.checkOnExisting(entityTable) && entityTable instanceof SDf57) { + kafkaMessenger.sendUserNotification(ObjectType.rgst, + "Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее", + Priority.HIGH); + } else { + table.injectEntity(entityTable); + } } if (DF_01.equals(currTable)){ kafkaMessenger.sendPairSdfRequest(table, currTable.getPrefix()); diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java index e675ac155..07d896ef5 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java @@ -34,6 +34,10 @@ public abstract class AbstractTable { public abstract T getEntity(SWTRecord record); + public String getPrefix() { + return prefix; + } + public void injectEntity(T obj) { if (map == null) { bootMap(); @@ -41,6 +45,10 @@ public abstract class AbstractTable { map.insert(obj); } + public boolean checkOnExisting(T obj){ + return true; + } + protected void bootMap() { map = (ImdgHazelcast) hazelcastService.getImdg(nameOfMap, clazz); } diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java index e31825c51..559919806 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java @@ -1,14 +1,17 @@ package ru.spcex.clearing.swt.importer.logic.data.tables; +import java.time.Instant; +import java.util.Collection; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ru.clearing.classes.statics.data.sdf.SDf20; import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; import ru.spcex.clearing.swt.importer.readers.SWTRecord; -import java.time.Instant; - -public class SDf20Table extends AbstractTable { - +public class SDf20Table extends AbstractTable implements WithTransactionNumber { + private final Logger log = LoggerFactory.getLogger(getClass()); private static final String PREFIX = ETable.S_DF_20.name(); private static final Class CLAZZ = SDf20.class; private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf20; @@ -37,8 +40,18 @@ public class SDf20Table extends AbstractTable { result.setSecurityName(record.getValFor35BTag()[1]); result.setSecurityType(record.getValFor35BTag()[2]); result.setNameCl(record.getValFor83DTag()[1]); - result.setTransactionNumber(record.getValFor23Tag()); return result; } + @Override + public boolean checkOnExisting(SDf20 obj) { + if (map == null) { + bootMap(); + } + Collection sDf20s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber())); + if (!sDf20s.isEmpty()) { + log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber()); + } + return sDf20s.isEmpty(); + } } \ No newline at end of file diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java index cb8038b50..076868aa6 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java @@ -1,19 +1,17 @@ package ru.spcex.clearing.swt.importer.logic.data.tables; +import java.time.Instant; +import java.util.Collection; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.clearing.classes.statics.data.sdf.SDf21; import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; import ru.spcex.clearing.swt.importer.readers.SWTRecord; -import ru.spcex.platform.imdg.api.predicate.ImdgPredicate; -import ru.spcex.platform.imdg.api.predicate.ImdgPredicateBuilder; -import java.time.Instant; - -public class SDf21Table extends AbstractTable { +public class SDf21Table extends AbstractTable implements WithTransactionNumber { private final Logger log = LoggerFactory.getLogger(getClass()); - private static final String PREFIX = ETable.S_DF_21.name(); private static final Class CLAZZ = SDf21.class; private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf21; @@ -46,22 +44,14 @@ public class SDf21Table extends AbstractTable { } @Override - public void injectEntity(SDf21 obj) { + public boolean checkOnExisting(SDf21 obj) { if (map == null) { bootMap(); } - ImdgPredicateBuilder pb = map.predicateBuilder(); - ImdgPredicate sameTransactionNumberAndOperationCode = pb.and( - pb.equals("transactionNumber", obj.getTransactionNumber()), - pb.equals("operationCode", obj.getOperationCode()) - ); - SDf21 firstObjectBySQL = map.getFirstObjectByPredicate( - sameTransactionNumberAndOperationCode - ); - if (firstObjectBySQL != null) { - log.warn("Skip insert by {}, sdf21.transactionNumber: {}", PREFIX, obj.getTransactionNumber()); - } else { - map.insert(obj); + Collection sDf21s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber())); + if (!sDf21s.isEmpty()) { + log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber()); } + return sDf21s.isEmpty(); } } \ No newline at end of file diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java new file mode 100644 index 000000000..42a6d4993 --- /dev/null +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java @@ -0,0 +1,4 @@ +package ru.spcex.clearing.swt.importer.logic.data.tables; + +public interface WithTransactionNumber { +} diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java index 9895d7fb2..aadb9dee0 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java @@ -1,17 +1,6 @@ package ru.spcex.clearing.swt.importer.logic.stages; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; -import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings; -import ru.spcex.clearing.swt.importer.logic.data.ResultContainer; -import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; -import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult; -import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable; -import ru.spcex.clearing.swt.importer.readers.SWTReader; -import ru.spcex.clearing.swt.importer.readers.SWTRecord; -import ru.spcex.clearing.swt.importer.readers.ValidationResult; -import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; -import ru.spcex.platform.utils.log.ExceptionUtils; +import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -20,8 +9,22 @@ import java.nio.charset.Charset; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; - -import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings; +import ru.spcex.clearing.swt.importer.logic.data.ResultContainer; +import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; +import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult; +import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable; +import ru.spcex.clearing.swt.importer.logic.data.tables.WithTransactionNumber; +import ru.spcex.clearing.swt.importer.readers.SWTReader; +import ru.spcex.clearing.swt.importer.readers.SWTRecord; +import ru.spcex.clearing.swt.importer.readers.ValidationResult; +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.enumeration.ObjectType; +import ru.spcex.platform.enumeration.Priority; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; +import ru.spcex.platform.utils.log.ExceptionUtils; /** * Заливка проверенных данных в базу @@ -68,8 +71,15 @@ public class ImportToDB extends Stage { List records = swtReader.getGroupRecords(); records.forEach(record -> { if (record.isNotEmpty()) { - table.injectEntity(table.getEntity(record)); - log.debug("{} entity stored.", counter.incrementAndGet()); + SpcexObjectBase entityTable = table.getEntity(record); + if (!table.checkOnExisting(entityTable) && table instanceof WithTransactionNumber) { + kafkaMessenger.sendUserNotification(ObjectType.rgst, + "Номер транзакции " + record.getValFor23Tag() + " в полученном " + table.getPrefix() + " уже был обработан ранее", + Priority.HIGH); + } else { + table.injectEntity(entityTable); + log.debug("{} entity stored.", counter.incrementAndGet()); + } } }); } while (swtReader.hasNextRecord()); diff --git a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java index 952469534..5433f3915 100644 --- a/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java +++ b/clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java @@ -84,7 +84,7 @@ public class SWTImportKafkaMessenger implements InitializingBean { // } } - private void sendUserNotification(ObjectType objectType, String comment, Priority priority) { + public void sendUserNotification(ObjectType objectType, String comment, Priority priority) { final String destination = Consts.NOTIFICATION_NEW; NotificationNewRequest request = new NotificationNewRequest(); //request.setObjectId(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java index 3789aef00..5a2800f81 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java @@ -34,7 +34,7 @@ public class ImporterImdgConfig { @Autowired @Bean - public ImdgProvider imdgProvider( + public HazelcastService imdgProvider( @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, ImportXMLServiceSettings settings diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java index a55a461c1..88797830f 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf01; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF01ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_01; + private static final Class CLAZZ = SDf01.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf01; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF01ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "CURR_CODE") private String currCode; @@ -50,10 +57,6 @@ public class DF01ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "FILE_TYPE") private String fileType; - public DF01ObjectTag() { - super(PREFIX); - } - @Override public SDf01 getSDfEntity() { SDf01 result = new SDf01(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java index b74b10e74..e40382446 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf04; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF04ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_04; + private static final Class CLAZZ = SDf04.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf04; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF04ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE") private String segType; @@ -61,10 +68,6 @@ public class DF04ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "IMP_RESULT") private String impResult; - public DF04ObjectTag() { - super(PREFIX); - } - @Override public SDf04 getSDfEntity() { SDf04 result = new SDf04(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java index 5da226924..334b0e653 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java @@ -8,12 +8,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf06; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF06ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_06; + private static final Class CLAZZ = SDf06.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf06; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF06ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT") private String account; @@ -54,10 +61,6 @@ public class DF06ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "PAY_VAL") private String payVal; - public DF06ObjectTag() { - super(PREFIX); - } - @Override public SDf06 getSDfEntity() { SDf06 result = new SDf06(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java index 45d1b3a20..7fc25418e 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf52; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF52ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_52; + private static final Class CLAZZ = SDf52.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf52; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF52ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT") private String account; @@ -32,10 +39,6 @@ public class DF52ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "ACC_TYPE") private String accType; - public DF52ObjectTag() { - super(PREFIX); - } - @Override public SDf52 getSDfEntity() { SDf52 result = new SDf52(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java index 508980cfc..ea7298862 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java @@ -7,12 +7,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Objects; import ru.clearing.classes.statics.data.sdf.SDf55; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF55ObjectTag extends ObjectTag { private static final ETable PREFIX = ETable.DF_55; + private static final Class CLAZZ = SDf55.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf55; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF55ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE") private String segType; @@ -123,10 +130,6 @@ public class DF55ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "SWIFT_INT") private String swiftInt; - public DF55ObjectTag() { - super(PREFIX); - } - @Override public SDf55 getSDfEntity() { SDf55 result = new SDf55(); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java index 4df10aa9c..2baa0c917 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java @@ -5,14 +5,26 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.time.Instant; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Collection; +import java.util.Map; import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ru.clearing.classes.statics.data.sdf.SDf57; +import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; public class DF57ObjectTag extends ObjectTag { + private final Logger log = LoggerFactory.getLogger(getClass()); private static final ETable PREFIX = ETable.DF_57; + private static final Class CLAZZ = SDf57.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf57; private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy"); + public DF57ObjectTag() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + @JacksonXmlProperty(isAttribute = true, localName = "ID") private Long id; @@ -116,10 +128,6 @@ public class DF57ObjectTag extends ObjectTag { @JacksonXmlProperty(isAttribute = true, localName = "KT_OUT") private String ktOut; - public DF57ObjectTag() { - super(PREFIX); - } - @Override public SDf57 getSDfEntity() { SDf57 result = new SDf57(); @@ -174,6 +182,18 @@ public class DF57ObjectTag extends ObjectTag { return result; } + @Override + public boolean checkOnExisting(SDf57 obj) { + if (map == null) { + bootMap(); + } + Collection sDf57s = map.getCollectionObjectsByFieldValues(Map.of("dbfId", obj.getDbfId())); + if (!sDf57s.isEmpty()) { + log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getDbfId()); + } + return sDf57s.isEmpty(); + } + public Long getId() { return id; } diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java index a746c5c82..533fecf8c 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; import ru.spcex.platform.classes.base.SpcexObjectBase; import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) @JsonSubTypes({ @@ -16,19 +17,25 @@ import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; @JsonSubTypes.Type(DF55ObjectTag.class), @JsonSubTypes.Type(DF57ObjectTag.class) }) -public abstract class ObjectTag { +public abstract class ObjectTag { private final ETable prefix; + private final Class clazz; + private final String nameOfMap; + protected HazelcastService hazelcastService; + protected ImdgHazelcast map; @JsonIgnore protected String fileName; @JsonIgnore protected Long generationId; - public ObjectTag(ETable prefix) { + public ObjectTag(ETable prefix, Class clazz, String nameOfMap) { this.prefix = prefix; + this.clazz = clazz; + this.nameOfMap = nameOfMap; } - public abstract T1 getSDfEntity(); + public abstract T getSDfEntity(); public ETable getPrefix() { return prefix; @@ -38,6 +45,10 @@ public abstract class ObjectTag { return fileName; } + public void setHazelcastService(HazelcastService hazelcastService) { + this.hazelcastService = hazelcastService; + } + public void setFileName(String fileName) { this.fileName = fileName; } @@ -50,14 +61,18 @@ public abstract class ObjectTag { this.generationId = generationId; } - public void insertEntity(ImdgHazelcast map) { - T1 obj = getSDfEntity(); - if (checkOnExisting(obj)) { - map.insert(obj); + public void injectEntity(T obj) { + if (map == null) { + bootMap(); } + map.insert(obj); } - protected boolean checkOnExisting(T1 obj) { + public boolean checkOnExisting(T obj) { return true; } + + protected void bootMap() { + map = (ImdgHazelcast) hazelcastService.getImdg(nameOfMap, clazz); + } } diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java index 35ce79a9f..a8ef05a82 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java @@ -1,36 +1,29 @@ package ru.spcex.clearing.xml.importer.logic.steps; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import ru.clearing.classes.statics.data.sdf.SDf57; import ru.spcex.clearing.xml.importer.logic.data.ResultContainer; import ru.spcex.clearing.xml.importer.logic.data.enums.ETable; import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult; import ru.spcex.clearing.xml.importer.logic.data.tags.DocumentTag; import ru.spcex.clearing.xml.importer.logic.data.tags.objects.ObjectTag; import ru.spcex.platform.classes.base.SpcexObjectBase; -import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; +import ru.spcex.platform.enumeration.ObjectType; +import ru.spcex.platform.enumeration.Priority; import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; @Component public class ImportToDB { private final Logger log = LoggerFactory.getLogger(getClass()); - private final XmlMapper xmlMapper; private final HazelcastService hazelcastService; private final XmlImportKafkaMessenger kafkaMessenger; - private final Map> mapOfTable; - public ImportToDB(@Qualifier("xmlMapper") XmlMapper xmlMapper, - HazelcastService hazelcastService, - XmlImportKafkaMessenger kafkaMessenger, - @Qualifier("mapOfTable") Map> mapOfTable) { - this.xmlMapper = xmlMapper; + public ImportToDB(HazelcastService hazelcastService, + XmlImportKafkaMessenger kafkaMessenger) { this.hazelcastService = hazelcastService; this.kafkaMessenger = kafkaMessenger; - this.mapOfTable = mapOfTable; } public StageResult process(ResultContainer resultContainer) { @@ -41,10 +34,18 @@ public class ImportToDB { String fileName = resultContainer.getXmlFile().getName(); Long fileId = hazelcastService.getImdgIdGenerator().nextId(); for (ObjectTag o : documentTag.getObjects()) { - ImdgHazelcast imdgMap = mapOfTable.get(o.getPrefix()); + o.setHazelcastService(hazelcastService); o.setFileName(fileName); o.setGenerationId(fileId); - o.insertEntity(imdgMap); + + SpcexObjectBase entityTable = o.getSDfEntity(); + if (!o.checkOnExisting(entityTable) && entityTable instanceof SDf57) { + kafkaMessenger.sendUserNotification(ObjectType.rgst, + "Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее", + Priority.HIGH); + } else { + o.injectEntity(entityTable); + } } if (ETable.DF_01.equals(currTable)) { diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java index 15c1f3bf0..166fcdb8f 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java @@ -1,5 +1,7 @@ package ru.spcex.clearing.xml.importer.logic.steps; +import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; + import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; @@ -9,7 +11,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.stereotype.Component; import ru.spcex.clearing.platform.messaging.domain.Consts; -import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF; import ru.spcex.clearing.platform.messaging.domain.cud.balance.StatementRequest; import ru.spcex.clearing.platform.messaging.domain.cud.clearing.Sdf04Request; import ru.spcex.clearing.platform.messaging.domain.cud.utilities.NotificationNewRequest; @@ -76,7 +77,7 @@ public class XmlImportKafkaMessenger implements InitializingBean { } } - private void sendUserNotification(ObjectType objectType, String comment, Priority priority) { + public void sendUserNotification(ObjectType objectType, String comment, Priority priority) { final String destination = Consts.NOTIFICATION_NEW; NotificationNewRequest request = new NotificationNewRequest(); request.setObjectType(objectType.getKey()); diff --git a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java index b803e8a70..4e0f4d602 100644 --- a/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java +++ b/clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java @@ -59,7 +59,11 @@ public class XMLImporterService { } public void processTable(ETable specificTable) { - log.info("adding import task {}", specificTable == null ? "without specific table" : specificTable); + if (specificTable == null) { + log.trace("adding import task without specific table"); + } else { + log.info("adding import task {}", specificTable); + } fileChecker.checkAndLoadSFTP(); executorService.execute(() -> { diff --git a/pom.xml b/pom.xml index e9f73aa7b..58c235554 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,8 @@ ${folder_root_clearing}/clearing-parent/registry-service ${folder_root_clearing}/clearing-parent/scheduler-service ${folder_root_clearing}/clearing-parent/gateway-api + ${folder_root_clearing}/clearing-parent/xml-exporter + ${folder_root_clearing}/clearing-parent/xml-importer 3.12.4 1.7.33