Merge remote-tracking branch 'origin/dev' into dev

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java

@@ -1,13 +1,12 @@
 package ru.spcex.clearing.dbf.importer.logic.data.tables;
 
+import java.math.BigDecimal;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import ru.spcex.platform.classes.base.SpcexObjectBase;
 import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
 import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
 
-import java.math.BigDecimal;
-
 public abstract class AbstractTable<T extends SpcexObjectBase> {
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final String prefix;
@@ -42,14 +41,13 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
         if (map == null) {
             bootMap();
         }
-        if (checkOnExisting(obj)) {
+        map.insert(obj);
-            map.insert(obj);
-        }
     }
 
-    protected boolean checkOnExisting(T obj){
+    public boolean checkOnExisting(T obj){
         return true;
     }
+
     protected void bootMap() {
         map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
     }

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java

@@ -1,15 +1,14 @@
 package ru.spcex.clearing.dbf.importer.logic.data.tables;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import ru.clearing.classes.statics.data.sdf.SDf57;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
-import ru.spcex.clearing.imdg.IMDGDistributedNames;
-
 import java.math.BigDecimal;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import ru.clearing.classes.statics.data.sdf.SDf57;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
 
 public class SDf57Table extends AbstractTable<SDf57> {
     private final Logger log = LoggerFactory.getLogger(getClass());
@@ -83,7 +82,7 @@ public class SDf57Table extends AbstractTable<SDf57> {
     }
 
     @Override
-    protected boolean checkOnExisting(SDf57 obj) {
+    public boolean checkOnExisting(SDf57 obj) {
         if (map == null) {
             bootMap();
         }

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java

@@ -1,5 +1,11 @@
 package ru.spcex.clearing.dbf.importer.logic.stages;
 
+import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
@@ -19,13 +25,6 @@ import ru.spcex.platform.enumeration.ObjectType;
 import ru.spcex.platform.enumeration.Priority;
 import ru.spcex.platform.enumeration.SdfTable;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
-
 @Component
 public class DbfImportKafkaMessenger implements InitializingBean {
     final Logger log = LoggerFactory.getLogger(getClass());
@@ -82,7 +81,7 @@ public class DbfImportKafkaMessenger implements InitializingBean {
         }
     }
 
-    private void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
+    public void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
         final String destination = Consts.NOTIFICATION_NEW;
         NotificationNewRequest request = new NotificationNewRequest();
         //request.setObjectId();

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java

@@ -1,23 +1,26 @@
 package ru.spcex.clearing.dbf.importer.logic.stages;
 
-import com.linuxense.javadbf.DBFReader;
+import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.stereotype.Component;
-import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings;
-import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
-import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable;
-import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
-import ru.spcex.platform.utils.log.ExceptionUtils;
 
+import com.linuxense.javadbf.DBFReader;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
 import java.util.Map;
-
+import org.springframework.beans.factory.annotation.Qualifier;
-import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01;
+import org.springframework.stereotype.Component;
+import ru.clearing.classes.statics.data.sdf.SDf57;
+import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings;
+import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
+import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable;
+import ru.spcex.platform.classes.base.SpcexObjectBase;
+import ru.spcex.platform.enumeration.ObjectType;
+import ru.spcex.platform.enumeration.Priority;
+import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
+import ru.spcex.platform.utils.log.ExceptionUtils;
 
 /**
  * Заливка проверенных данных в базу
@@ -58,7 +61,14 @@ public class ImportToDB extends Stage {
                     log.warn("record index {} null", i);
                     continue;
                 }
-                table.injectEntity(table.getEntity(entity));
+                SpcexObjectBase entityTable = table.getEntity(entity);
+                if (!table.checkOnExisting(entityTable) && entityTable instanceof SDf57) {
+                    kafkaMessenger.sendUserNotification(ObjectType.rgst,
+                        "Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее",
+                        Priority.HIGH);
+                } else {
+                    table.injectEntity(entityTable);
+                }
             }
             if (DF_01.equals(currTable)){
                 kafkaMessenger.sendPairSdfRequest(table, currTable.getPrefix());

clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java

@@ -34,6 +34,10 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
 
     public abstract T getEntity(SWTRecord record);
 
+    public String getPrefix() {
+        return prefix;
+    }
+
     public void injectEntity(T obj) {
         if (map == null) {
             bootMap();
@@ -41,6 +45,10 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
         map.insert(obj);
     }
 
+    public boolean checkOnExisting(T obj){
+        return true;
+    }
+
     protected void bootMap() {
         map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
     }

clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java

@@ -1,14 +1,17 @@
 package ru.spcex.clearing.swt.importer.logic.data.tables;
 
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import ru.clearing.classes.statics.data.sdf.SDf20;
 import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
 import ru.spcex.clearing.swt.importer.readers.SWTRecord;
 
-import java.time.Instant;
+public class SDf20Table extends AbstractTable<SDf20> implements WithTransactionNumber {
-
+    private final Logger log = LoggerFactory.getLogger(getClass());
-public class SDf20Table extends AbstractTable<SDf20> {
-
     private static final String PREFIX = ETable.S_DF_20.name();
     private static final Class<SDf20> CLAZZ = SDf20.class;
     private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf20;
@@ -37,8 +40,18 @@ public class SDf20Table extends AbstractTable<SDf20> {
         result.setSecurityName(record.getValFor35BTag()[1]);
         result.setSecurityType(record.getValFor35BTag()[2]);
         result.setNameCl(record.getValFor83DTag()[1]);
-        result.setTransactionNumber(record.getValFor23Tag());
         return result;
     }
 
+    @Override
+    public boolean checkOnExisting(SDf20 obj) {
+        if (map == null) {
+            bootMap();
+        }
+        Collection<SDf20> sDf20s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber()));
+        if (!sDf20s.isEmpty()) {
+            log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber());
+        }
+        return sDf20s.isEmpty();
+    }
 }

clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java

@@ -1,19 +1,17 @@
 package ru.spcex.clearing.swt.importer.logic.data.tables;
 
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import ru.clearing.classes.statics.data.sdf.SDf21;
 import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
 import ru.spcex.clearing.swt.importer.readers.SWTRecord;
-import ru.spcex.platform.imdg.api.predicate.ImdgPredicate;
-import ru.spcex.platform.imdg.api.predicate.ImdgPredicateBuilder;
 
-import java.time.Instant;
+public class SDf21Table extends AbstractTable<SDf21> implements WithTransactionNumber {
-
-public class SDf21Table extends AbstractTable<SDf21> {
     private final Logger log = LoggerFactory.getLogger(getClass());
-
     private static final String PREFIX = ETable.S_DF_21.name();
     private static final Class<SDf21> CLAZZ = SDf21.class;
     private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf21;
@@ -46,22 +44,14 @@ public class SDf21Table extends AbstractTable<SDf21> {
     }
 
     @Override
-    public void injectEntity(SDf21 obj) {
+    public boolean checkOnExisting(SDf21 obj) {
         if (map == null) {
             bootMap();
         }
-        ImdgPredicateBuilder pb = map.predicateBuilder();
+        Collection<SDf21> sDf21s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber()));
-        ImdgPredicate sameTransactionNumberAndOperationCode = pb.and(
+        if (!sDf21s.isEmpty()) {
-                pb.equals("transactionNumber", obj.getTransactionNumber()),
+            log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber());
-                pb.equals("operationCode", obj.getOperationCode())
-        );
-        SDf21 firstObjectBySQL = map.getFirstObjectByPredicate(
-                sameTransactionNumberAndOperationCode
-        );
-        if (firstObjectBySQL != null) {
-            log.warn("Skip insert by {}, sdf21.transactionNumber: {}", PREFIX, obj.getTransactionNumber());
-        } else {
-            map.insert(obj);
         }
+        return sDf21s.isEmpty();
     }
 }

clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java

@@ -0,0 +1,4 @@
+package ru.spcex.clearing.swt.importer.logic.data.tables;
+
+public interface WithTransactionNumber {
+}

clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java

@@ -1,17 +1,6 @@
 package ru.spcex.clearing.swt.importer.logic.stages;
 
-import org.springframework.beans.factory.annotation.Qualifier;
+import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS;
-import org.springframework.stereotype.Component;
-import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
-import ru.spcex.clearing.swt.importer.logic.data.ResultContainer;
-import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
-import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
-import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
-import ru.spcex.clearing.swt.importer.readers.SWTReader;
-import ru.spcex.clearing.swt.importer.readers.SWTRecord;
-import ru.spcex.clearing.swt.importer.readers.ValidationResult;
-import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
-import ru.spcex.platform.utils.log.ExceptionUtils;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -20,8 +9,22 @@ import java.nio.charset.Charset;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
-
+import org.springframework.beans.factory.annotation.Qualifier;
-import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS;
+import org.springframework.stereotype.Component;
+import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
+import ru.spcex.clearing.swt.importer.logic.data.ResultContainer;
+import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
+import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
+import ru.spcex.clearing.swt.importer.logic.data.tables.WithTransactionNumber;
+import ru.spcex.clearing.swt.importer.readers.SWTReader;
+import ru.spcex.clearing.swt.importer.readers.SWTRecord;
+import ru.spcex.clearing.swt.importer.readers.ValidationResult;
+import ru.spcex.platform.classes.base.SpcexObjectBase;
+import ru.spcex.platform.enumeration.ObjectType;
+import ru.spcex.platform.enumeration.Priority;
+import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
+import ru.spcex.platform.utils.log.ExceptionUtils;
 
 /**
  * Заливка проверенных данных в базу
@@ -68,8 +71,15 @@ public class ImportToDB extends Stage {
                 List<SWTRecord> records = swtReader.getGroupRecords();
                 records.forEach(record -> {
                     if (record.isNotEmpty()) {
-                        table.injectEntity(table.getEntity(record));
+                        SpcexObjectBase entityTable = table.getEntity(record);
-                        log.debug("{} entity stored.", counter.incrementAndGet());
+                        if (!table.checkOnExisting(entityTable) && table instanceof WithTransactionNumber) {
+                            kafkaMessenger.sendUserNotification(ObjectType.rgst,
+                                "Номер транзакции " + record.getValFor23Tag() + " в полученном " + table.getPrefix() + " уже был обработан ранее",
+                                Priority.HIGH);
+                        } else {
+                            table.injectEntity(entityTable);
+                            log.debug("{} entity stored.", counter.incrementAndGet());
+                        }
                     }
                 });
             } while (swtReader.hasNextRecord());

clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java

@@ -84,7 +84,7 @@ public class SWTImportKafkaMessenger implements InitializingBean {
 //        }
     }
 
-    private void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
+    public void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
         final String destination = Consts.NOTIFICATION_NEW;
         NotificationNewRequest request = new NotificationNewRequest();
         //request.setObjectId();

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java

@@ -34,7 +34,7 @@ public class ImporterImdgConfig {
 
     @Autowired
     @Bean
-    public ImdgProvider imdgProvider(
+    public HazelcastService imdgProvider(
         @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
         @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
         ImportXMLServiceSettings settings

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java

@@ -7,12 +7,19 @@ import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Objects;
 import ru.clearing.classes.statics.data.sdf.SDf01;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 
 public class DF01ObjectTag extends ObjectTag<SDf01> {
     private static final ETable PREFIX = ETable.DF_01;
+    private static final Class<SDf01> CLAZZ = SDf01.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf01;
     private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
 
+    public DF01ObjectTag() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
     @JacksonXmlProperty(isAttribute = true, localName = "CURR_CODE")
     private String currCode;
 
@@ -50,10 +57,6 @@ public class DF01ObjectTag extends ObjectTag<SDf01> {
     @JacksonXmlProperty(isAttribute = true, localName = "FILE_TYPE")
     private String fileType;
 
-    public DF01ObjectTag() {
-        super(PREFIX);
-    }
-
     @Override
     public SDf01 getSDfEntity() {
         SDf01 result = new SDf01();

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java

@@ -7,12 +7,19 @@ import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Objects;
 import ru.clearing.classes.statics.data.sdf.SDf04;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 
 public class DF04ObjectTag extends ObjectTag<SDf04> {
     private static final ETable PREFIX = ETable.DF_04;
+    private static final Class<SDf04> CLAZZ = SDf04.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf04;
     private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
 
+    public DF04ObjectTag() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
     @JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE")
     private String segType;
 
@@ -61,10 +68,6 @@ public class DF04ObjectTag extends ObjectTag<SDf04> {
     @JacksonXmlProperty(isAttribute = true, localName = "IMP_RESULT")
     private String impResult;
 
-    public DF04ObjectTag() {
-        super(PREFIX);
-    }
-
     @Override
     public SDf04 getSDfEntity() {
         SDf04 result = new SDf04();

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java

@@ -8,12 +8,19 @@ import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Objects;
 import ru.clearing.classes.statics.data.sdf.SDf06;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 
 public class DF06ObjectTag extends ObjectTag<SDf06> {
     private static final ETable PREFIX = ETable.DF_06;
+    private static final Class<SDf06> CLAZZ = SDf06.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf06;
     private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
 
+    public DF06ObjectTag() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
     @JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT")
     private String account;
 
@@ -54,10 +61,6 @@ public class DF06ObjectTag extends ObjectTag<SDf06> {
     @JacksonXmlProperty(isAttribute = true, localName = "PAY_VAL")
     private String payVal;
 
-    public DF06ObjectTag() {
-        super(PREFIX);
-    }
-
     @Override
     public SDf06 getSDfEntity() {
         SDf06 result = new SDf06();

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java

@@ -7,12 +7,19 @@ import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Objects;
 import ru.clearing.classes.statics.data.sdf.SDf52;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 
 public class DF52ObjectTag extends ObjectTag<SDf52> {
     private static final ETable PREFIX = ETable.DF_52;
+    private static final Class<SDf52> CLAZZ = SDf52.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf52;
     private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
 
+    public DF52ObjectTag() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
     @JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT")
     private String account;
 
@@ -32,10 +39,6 @@ public class DF52ObjectTag extends ObjectTag<SDf52> {
     @JacksonXmlProperty(isAttribute = true, localName = "ACC_TYPE")
     private String accType;
 
-    public DF52ObjectTag() {
-        super(PREFIX);
-    }
-
     @Override
     public SDf52 getSDfEntity() {
         SDf52 result = new SDf52();

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java

@@ -7,12 +7,19 @@ import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Objects;
 import ru.clearing.classes.statics.data.sdf.SDf55;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 
 public class DF55ObjectTag extends ObjectTag<SDf55> {
     private static final ETable PREFIX = ETable.DF_55;
+    private static final Class<SDf55> CLAZZ = SDf55.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf55;
     private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
 
+    public DF55ObjectTag() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
     @JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE")
     private String segType;
 
@@ -123,10 +130,6 @@ public class DF55ObjectTag extends ObjectTag<SDf55> {
     @JacksonXmlProperty(isAttribute = true, localName = "SWIFT_INT")
     private String swiftInt;
 
-    public DF55ObjectTag() {
-        super(PREFIX);
-    }
-
     @Override
     public SDf55 getSDfEntity() {
         SDf55 result = new SDf55();

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java

@@ -5,14 +5,26 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import java.util.Collection;
+import java.util.Map;
 import java.util.Objects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import ru.clearing.classes.statics.data.sdf.SDf57;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 
 public class DF57ObjectTag extends ObjectTag<SDf57> {
+    private final Logger log = LoggerFactory.getLogger(getClass());
     private static final ETable PREFIX = ETable.DF_57;
+    private static final Class<SDf57> CLAZZ = SDf57.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf57;
     private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
 
+    public DF57ObjectTag() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
     @JacksonXmlProperty(isAttribute = true, localName = "ID")
     private Long id;
 
@@ -116,10 +128,6 @@ public class DF57ObjectTag extends ObjectTag<SDf57> {
     @JacksonXmlProperty(isAttribute = true, localName = "KT_OUT")
     private String ktOut;
 
-    public DF57ObjectTag() {
-        super(PREFIX);
-    }
-
     @Override
     public SDf57 getSDfEntity() {
         SDf57 result = new SDf57();
@@ -174,6 +182,18 @@ public class DF57ObjectTag extends ObjectTag<SDf57> {
         return result;
     }
 
+    @Override
+    public boolean checkOnExisting(SDf57 obj) {
+        if (map == null) {
+            bootMap();
+        }
+        Collection<SDf57> sDf57s = map.getCollectionObjectsByFieldValues(Map.of("dbfId", obj.getDbfId()));
+        if (!sDf57s.isEmpty()) {
+            log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getDbfId());
+        }
+        return sDf57s.isEmpty();
+    }
+
     public Long getId() {
         return id;
     }

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java

@@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 import ru.spcex.platform.classes.base.SpcexObjectBase;
 import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
+import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION)
 @JsonSubTypes({
@@ -16,19 +17,25 @@ import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
     @JsonSubTypes.Type(DF55ObjectTag.class),
     @JsonSubTypes.Type(DF57ObjectTag.class)
 })
-public abstract class ObjectTag<T1 extends SpcexObjectBase> {
+public abstract class ObjectTag<T extends SpcexObjectBase> {
     private final ETable prefix;
+    private final Class<T> clazz;
+    private final String nameOfMap;
+    protected HazelcastService hazelcastService;
+    protected ImdgHazelcast<T> map;
 
     @JsonIgnore
     protected String fileName;
     @JsonIgnore
     protected Long generationId;
 
-    public ObjectTag(ETable prefix) {
+    public ObjectTag(ETable prefix, Class<T> clazz, String nameOfMap) {
         this.prefix = prefix;
+        this.clazz = clazz;
+        this.nameOfMap = nameOfMap;
     }
 
-    public abstract T1 getSDfEntity();
+    public abstract T getSDfEntity();
 
     public ETable getPrefix() {
         return prefix;
@@ -38,6 +45,10 @@ public abstract class ObjectTag<T1 extends SpcexObjectBase> {
         return fileName;
     }
 
+    public void setHazelcastService(HazelcastService hazelcastService) {
+        this.hazelcastService = hazelcastService;
+    }
+
     public void setFileName(String fileName) {
         this.fileName = fileName;
     }
@@ -50,14 +61,18 @@ public abstract class ObjectTag<T1 extends SpcexObjectBase> {
         this.generationId = generationId;
     }
 
-    public void insertEntity(ImdgHazelcast<T1> map) {
+    public void injectEntity(T obj) {
-        T1 obj = getSDfEntity();
+        if (map == null) {
-        if (checkOnExisting(obj)) {
+            bootMap();
-            map.insert(obj);
         }
+        map.insert(obj);
     }
 
-    protected boolean checkOnExisting(T1 obj) {
+    public boolean checkOnExisting(T obj) {
         return true;
     }
+
+    protected void bootMap() {
+        map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
+    }
 }

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java

@@ -1,36 +1,29 @@
 package ru.spcex.clearing.xml.importer.logic.steps;
 
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Component;
+import ru.clearing.classes.statics.data.sdf.SDf57;
 import ru.spcex.clearing.xml.importer.logic.data.ResultContainer;
 import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
 import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult;
 import ru.spcex.clearing.xml.importer.logic.data.tags.DocumentTag;
 import ru.spcex.clearing.xml.importer.logic.data.tags.objects.ObjectTag;
 import ru.spcex.platform.classes.base.SpcexObjectBase;
-import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
+import ru.spcex.platform.enumeration.ObjectType;
+import ru.spcex.platform.enumeration.Priority;
 import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
 
 @Component
 public class ImportToDB {
     private final Logger log = LoggerFactory.getLogger(getClass());
-    private final XmlMapper xmlMapper;
     private final HazelcastService hazelcastService;
     private final XmlImportKafkaMessenger kafkaMessenger;
-    private final Map<ETable, ImdgHazelcast<?>> mapOfTable;
 
-    public ImportToDB(@Qualifier("xmlMapper") XmlMapper xmlMapper,
+    public ImportToDB(HazelcastService hazelcastService,
-                      HazelcastService hazelcastService,
+                      XmlImportKafkaMessenger kafkaMessenger) {
-                      XmlImportKafkaMessenger kafkaMessenger,
-                      @Qualifier("mapOfTable") Map<ETable, ImdgHazelcast<?>> mapOfTable) {
-        this.xmlMapper = xmlMapper;
         this.hazelcastService = hazelcastService;
         this.kafkaMessenger = kafkaMessenger;
-        this.mapOfTable = mapOfTable;
     }
 
     public StageResult process(ResultContainer resultContainer) {
@@ -41,10 +34,18 @@ public class ImportToDB {
         String fileName = resultContainer.getXmlFile().getName();
         Long fileId = hazelcastService.getImdgIdGenerator().nextId();
         for (ObjectTag o : documentTag.getObjects()) {
-            ImdgHazelcast<? extends SpcexObjectBase> imdgMap = mapOfTable.get(o.getPrefix());
+            o.setHazelcastService(hazelcastService);
             o.setFileName(fileName);
             o.setGenerationId(fileId);
-            o.insertEntity(imdgMap);
+
+            SpcexObjectBase entityTable = o.getSDfEntity();
+            if (!o.checkOnExisting(entityTable) && entityTable instanceof SDf57) {
+                kafkaMessenger.sendUserNotification(ObjectType.rgst,
+                    "Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее",
+                    Priority.HIGH);
+            } else {
+                o.injectEntity(entityTable);
+            }
         }
 
         if (ETable.DF_01.equals(currTable)) {

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java

@@ -1,5 +1,7 @@
 package ru.spcex.clearing.xml.importer.logic.steps;
 
+import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
+
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;
@@ -9,7 +11,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.stereotype.Component;
 import ru.spcex.clearing.platform.messaging.domain.Consts;
-import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
 import ru.spcex.clearing.platform.messaging.domain.cud.balance.StatementRequest;
 import ru.spcex.clearing.platform.messaging.domain.cud.clearing.Sdf04Request;
 import ru.spcex.clearing.platform.messaging.domain.cud.utilities.NotificationNewRequest;
@@ -76,7 +77,7 @@ public class XmlImportKafkaMessenger implements InitializingBean {
         }
     }
 
-    private void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
+    public void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
         final String destination = Consts.NOTIFICATION_NEW;
         NotificationNewRequest request = new NotificationNewRequest();
         request.setObjectType(objectType.getKey());

clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java

@@ -59,7 +59,11 @@ public class XMLImporterService {
     }
 
     public void processTable(ETable specificTable) {
-        log.info("adding import task {}", specificTable == null ? "without specific table" : specificTable);
+        if (specificTable == null) {
+            log.trace("adding import task without specific table");
+        } else {
+            log.info("adding import task {}", specificTable);
+        }
 
         fileChecker.checkAndLoadSFTP();
         executorService.execute(() -> {