treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
ialbert 2024-08-08 16:15:41 +03:00
commit c9fd35d1c3
28 changed files with 293 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.gitlab-ci.yml
View file

@ -23,6 +23,9 @@ preflight:
expire_in: 1 day
paths:
- clearing_test
only:
- dev
resource_group: clearing-test
build:
image: ${DOCKER_PROXY}/maven:latest
@ -34,6 +37,9 @@ build:
expire_in: 1 day
paths:
- /builds/mfd/clearing/z-distr/target/clearing
only:
- dev
resource_group: clearing-test
test:
stage: test
@ -50,4 +56,7 @@ test:
expire_in: 1 day
when: always
paths:
- $CI_PROJECT_DIR/clr_test/mnt
- $CI_PROJECT_DIR/clr_test/mnt
only:
- dev
resource_group: clearing-test

10
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java
View file

@ -1,13 +1,12 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import java.math.BigDecimal;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import java.math.BigDecimal;
public abstract class AbstractTable<T extends SpcexObjectBase> {
private final Logger log = LoggerFactory.getLogger(getClass());
private final String prefix;
@ -42,14 +41,13 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
if (map == null) {
bootMap();
}
if (checkOnExisting(obj)) {
map.insert(obj);
}
map.insert(obj);
}
protected boolean checkOnExisting(T obj){
public boolean checkOnExisting(T obj){
return true;
}
protected void bootMap() {
map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
}

13
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf57Table.java
View file

@ -1,15 +1,14 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.clearing.classes.statics.data.sdf.SDf57;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.Collection;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.clearing.classes.statics.data.sdf.SDf57;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
public class SDf57Table extends AbstractTable<SDf57> {
private final Logger log = LoggerFactory.getLogger(getClass());
@ -83,7 +82,7 @@ public class SDf57Table extends AbstractTable<SDf57> {
}
@Override
protected boolean checkOnExisting(SDf57 obj) {
public boolean checkOnExisting(SDf57 obj) {
if (map == null) {
bootMap();
}

15
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/DbfImportKafkaMessenger.java
View file

@ -1,5 +1,11 @@
package ru.spcex.clearing.dbf.importer.logic.stages;
import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
@ -19,13 +25,6 @@ import ru.spcex.platform.enumeration.ObjectType;
import ru.spcex.platform.enumeration.Priority;
import ru.spcex.platform.enumeration.SdfTable;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
@Component
public class DbfImportKafkaMessenger implements InitializingBean {
final Logger log = LoggerFactory.getLogger(getClass());
@ -82,7 +81,7 @@ public class DbfImportKafkaMessenger implements InitializingBean {
}
}
private void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
public void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
final String destination = Consts.NOTIFICATION_NEW;
NotificationNewRequest request = new NotificationNewRequest();
//request.setObjectId();

36
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java
View file

@ -1,23 +1,26 @@
package ru.spcex.clearing.dbf.importer.logic.stages;
import com.linuxense.javadbf.DBFReader;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings;
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import ru.spcex.platform.utils.log.ExceptionUtils;
import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01;
import com.linuxense.javadbf.DBFReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Map;
import static ru.spcex.clearing.dbf.importer.logic.data.enums.ETable.DF_01;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.clearing.classes.statics.data.sdf.SDf57;
import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings;
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.enumeration.ObjectType;
import ru.spcex.platform.enumeration.Priority;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import ru.spcex.platform.utils.log.ExceptionUtils;
/**
* Заливка проверенных данных в базу
@ -58,7 +61,14 @@ public class ImportToDB extends Stage {
log.warn("record index {} null", i);
continue;
}
table.injectEntity(table.getEntity(entity));
SpcexObjectBase entityTable = table.getEntity(entity);
if (!table.checkOnExisting(entityTable) && entityTable instanceof SDf57) {
kafkaMessenger.sendUserNotification(ObjectType.rgst,
"Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее",
Priority.HIGH);
} else {
table.injectEntity(entityTable);
}
}
if (DF_01.equals(currTable)){
kafkaMessenger.sendPairSdfRequest(table, currTable.getPrefix());

8
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java
View file

@ -34,6 +34,10 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
public abstract T getEntity(SWTRecord record);
public String getPrefix() {
return prefix;
}
public void injectEntity(T obj) {
if (map == null) {
bootMap();
@ -41,6 +45,10 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
map.insert(obj);
}
public boolean checkOnExisting(T obj){
return true;
}
protected void bootMap() {
map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
}

23
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf20Table.java
View file

@ -1,14 +1,17 @@
package ru.spcex.clearing.swt.importer.logic.data.tables;
import java.time.Instant;
import java.util.Collection;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.clearing.classes.statics.data.sdf.SDf20;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
import ru.spcex.clearing.swt.importer.readers.SWTRecord;
import java.time.Instant;
public class SDf20Table extends AbstractTable<SDf20> {
public class SDf20Table extends AbstractTable<SDf20> implements WithTransactionNumber {
private final Logger log = LoggerFactory.getLogger(getClass());
private static final String PREFIX = ETable.S_DF_20.name();
private static final Class<SDf20> CLAZZ = SDf20.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf20;
@ -37,8 +40,18 @@ public class SDf20Table extends AbstractTable<SDf20> {
result.setSecurityName(record.getValFor35BTag()[1]);
result.setSecurityType(record.getValFor35BTag()[2]);
result.setNameCl(record.getValFor83DTag()[1]);
result.setTransactionNumber(record.getValFor23Tag());
return result;
}
@Override
public boolean checkOnExisting(SDf20 obj) {
if (map == null) {
bootMap();
}
Collection<SDf20> sDf20s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber()));
if (!sDf20s.isEmpty()) {
log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber());
}
return sDf20s.isEmpty();
}
}

28
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf21Table.java
View file

@ -1,19 +1,17 @@
package ru.spcex.clearing.swt.importer.logic.data.tables;
import java.time.Instant;
import java.util.Collection;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.clearing.classes.statics.data.sdf.SDf21;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
import ru.spcex.clearing.swt.importer.readers.SWTRecord;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicate;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicateBuilder;
import java.time.Instant;
public class SDf21Table extends AbstractTable<SDf21> {
public class SDf21Table extends AbstractTable<SDf21> implements WithTransactionNumber {
private final Logger log = LoggerFactory.getLogger(getClass());
private static final String PREFIX = ETable.S_DF_21.name();
private static final Class<SDf21> CLAZZ = SDf21.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf21;
@ -46,22 +44,14 @@ public class SDf21Table extends AbstractTable<SDf21> {
}
@Override
public void injectEntity(SDf21 obj) {
public boolean checkOnExisting(SDf21 obj) {
if (map == null) {
bootMap();
}
ImdgPredicateBuilder pb = map.predicateBuilder();
ImdgPredicate sameTransactionNumberAndOperationCode = pb.and(
pb.equals("transactionNumber", obj.getTransactionNumber()),
pb.equals("operationCode", obj.getOperationCode())
);
SDf21 firstObjectBySQL = map.getFirstObjectByPredicate(
sameTransactionNumberAndOperationCode
);
if (firstObjectBySQL != null) {
log.warn("Skip insert by {}, sdf21.transactionNumber: {}", PREFIX, obj.getTransactionNumber());
} else {
map.insert(obj);
Collection<SDf21> sDf21s = map.getCollectionObjectsByFieldValues(Map.of("transactionNumber", obj.getTransactionNumber()));
if (!sDf21s.isEmpty()) {
log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getTransactionNumber());
}
return sDf21s.isEmpty();
}
}

4
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/WithTransactionNumber.java Normal file
View file

@ -0,0 +1,4 @@
package ru.spcex.clearing.swt.importer.logic.data.tables;
public interface WithTransactionNumber {
}

42
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java
View file

@ -1,17 +1,6 @@
package ru.spcex.clearing.swt.importer.logic.stages;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
import ru.spcex.clearing.swt.importer.logic.data.ResultContainer;
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
import ru.spcex.clearing.swt.importer.readers.SWTReader;
import ru.spcex.clearing.swt.importer.readers.SWTRecord;
import ru.spcex.clearing.swt.importer.readers.ValidationResult;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import ru.spcex.platform.utils.log.ExceptionUtils;
import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -20,8 +9,22 @@ import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static ru.spcex.clearing.swt.importer.readers.ValidationResult.SUCSESS;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
import ru.spcex.clearing.swt.importer.logic.data.ResultContainer;
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
import ru.spcex.clearing.swt.importer.logic.data.tables.WithTransactionNumber;
import ru.spcex.clearing.swt.importer.readers.SWTReader;
import ru.spcex.clearing.swt.importer.readers.SWTRecord;
import ru.spcex.clearing.swt.importer.readers.ValidationResult;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.enumeration.ObjectType;
import ru.spcex.platform.enumeration.Priority;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import ru.spcex.platform.utils.log.ExceptionUtils;
/**
* Заливка проверенных данных в базу
@ -68,8 +71,15 @@ public class ImportToDB extends Stage {
List<SWTRecord> records = swtReader.getGroupRecords();
records.forEach(record -> {
if (record.isNotEmpty()) {
table.injectEntity(table.getEntity(record));
log.debug("{} entity stored.", counter.incrementAndGet());
SpcexObjectBase entityTable = table.getEntity(record);
if (!table.checkOnExisting(entityTable) && table instanceof WithTransactionNumber) {
kafkaMessenger.sendUserNotification(ObjectType.rgst,
"Номер транзакции " + record.getValFor23Tag() + " в полученном " + table.getPrefix() + " уже был обработан ранее",
Priority.HIGH);
} else {
table.injectEntity(entityTable);
log.debug("{} entity stored.", counter.incrementAndGet());
}
}
});
} while (swtReader.hasNextRecord());

2
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/SWTImportKafkaMessenger.java
View file

@ -84,7 +84,7 @@ public class SWTImportKafkaMessenger implements InitializingBean {
// }
}
private void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
public void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
final String destination = Consts.NOTIFICATION_NEW;
NotificationNewRequest request = new NotificationNewRequest();
//request.setObjectId();

2
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/ImporterImdgConfig.java
View file

@ -34,7 +34,7 @@ public class ImporterImdgConfig {
@Autowired
@Bean
public ImdgProvider imdgProvider(
public HazelcastService imdgProvider(
@Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
@Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
ImportXMLServiceSettings settings

11
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF01ObjectTag.java
View file

@ -7,12 +7,19 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
import ru.clearing.classes.statics.data.sdf.SDf01;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
public class DF01ObjectTag extends ObjectTag<SDf01> {
private static final ETable PREFIX = ETable.DF_01;
private static final Class<SDf01> CLAZZ = SDf01.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf01;
private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
public DF01ObjectTag() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@JacksonXmlProperty(isAttribute = true, localName = "CURR_CODE")
private String currCode;
@ -50,10 +57,6 @@ public class DF01ObjectTag extends ObjectTag<SDf01> {
@JacksonXmlProperty(isAttribute = true, localName = "FILE_TYPE")
private String fileType;
public DF01ObjectTag() {
super(PREFIX);
}
@Override
public SDf01 getSDfEntity() {
SDf01 result = new SDf01();

11
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF04ObjectTag.java
View file

@ -7,12 +7,19 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
import ru.clearing.classes.statics.data.sdf.SDf04;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
public class DF04ObjectTag extends ObjectTag<SDf04> {
private static final ETable PREFIX = ETable.DF_04;
private static final Class<SDf04> CLAZZ = SDf04.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf04;
private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
public DF04ObjectTag() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE")
private String segType;
@ -61,10 +68,6 @@ public class DF04ObjectTag extends ObjectTag<SDf04> {
@JacksonXmlProperty(isAttribute = true, localName = "IMP_RESULT")
private String impResult;
public DF04ObjectTag() {
super(PREFIX);
}
@Override
public SDf04 getSDfEntity() {
SDf04 result = new SDf04();

11
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF06ObjectTag.java
View file

@ -8,12 +8,19 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
import ru.clearing.classes.statics.data.sdf.SDf06;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
public class DF06ObjectTag extends ObjectTag<SDf06> {
private static final ETable PREFIX = ETable.DF_06;
private static final Class<SDf06> CLAZZ = SDf06.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf06;
private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
public DF06ObjectTag() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT")
private String account;
@ -54,10 +61,6 @@ public class DF06ObjectTag extends ObjectTag<SDf06> {
@JacksonXmlProperty(isAttribute = true, localName = "PAY_VAL")
private String payVal;
public DF06ObjectTag() {
super(PREFIX);
}
@Override
public SDf06 getSDfEntity() {
SDf06 result = new SDf06();

11
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF52ObjectTag.java
View file

@ -7,12 +7,19 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
import ru.clearing.classes.statics.data.sdf.SDf52;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
public class DF52ObjectTag extends ObjectTag<SDf52> {
private static final ETable PREFIX = ETable.DF_52;
private static final Class<SDf52> CLAZZ = SDf52.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf52;
private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
public DF52ObjectTag() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT")
private String account;
@ -32,10 +39,6 @@ public class DF52ObjectTag extends ObjectTag<SDf52> {
@JacksonXmlProperty(isAttribute = true, localName = "ACC_TYPE")
private String accType;
public DF52ObjectTag() {
super(PREFIX);
}
@Override
public SDf52 getSDfEntity() {
SDf52 result = new SDf52();

11
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF55ObjectTag.java
View file

@ -7,12 +7,19 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
import ru.clearing.classes.statics.data.sdf.SDf55;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
public class DF55ObjectTag extends ObjectTag<SDf55> {
private static final ETable PREFIX = ETable.DF_55;
private static final Class<SDf55> CLAZZ = SDf55.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf55;
private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
public DF55ObjectTag() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@JacksonXmlProperty(isAttribute = true, localName = "SEG_TYPE")
private String segType;
@ -123,10 +130,6 @@ public class DF55ObjectTag extends ObjectTag<SDf55> {
@JacksonXmlProperty(isAttribute = true, localName = "SWIFT_INT")
private String swiftInt;
public DF55ObjectTag() {
super(PREFIX);
}
@Override
public SDf55 getSDfEntity() {
SDf55 result = new SDf55();

28
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/DF57ObjectTag.java
View file

@ -5,14 +5,26 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import java.time.Instant;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collection;
import java.util.Map;
import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.clearing.classes.statics.data.sdf.SDf57;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
public class DF57ObjectTag extends ObjectTag<SDf57> {
private final Logger log = LoggerFactory.getLogger(getClass());
private static final ETable PREFIX = ETable.DF_57;
private static final Class<SDf57> CLAZZ = SDf57.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf57;
private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy");
public DF57ObjectTag() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@JacksonXmlProperty(isAttribute = true, localName = "ID")
private Long id;
@ -116,10 +128,6 @@ public class DF57ObjectTag extends ObjectTag<SDf57> {
@JacksonXmlProperty(isAttribute = true, localName = "KT_OUT")
private String ktOut;
public DF57ObjectTag() {
super(PREFIX);
}
@Override
public SDf57 getSDfEntity() {
SDf57 result = new SDf57();
@ -174,6 +182,18 @@ public class DF57ObjectTag extends ObjectTag<SDf57> {
return result;
}
@Override
public boolean checkOnExisting(SDf57 obj) {
if (map == null) {
bootMap();
}
Collection<SDf57> sDf57s = map.getCollectionObjectsByFieldValues(Map.of("dbfId", obj.getDbfId()));
if (!sDf57s.isEmpty()) {
log.warn("Skip insert by {}, sdf57.dbfId: {}", PREFIX, obj.getDbfId());
}
return sDf57s.isEmpty();
}
public Long getId() {
return id;
}

31
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/data/tags/objects/ObjectTag.java
View file

@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
@JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION)
@JsonSubTypes({
@ -16,19 +17,25 @@ import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
@JsonSubTypes.Type(DF55ObjectTag.class),
@JsonSubTypes.Type(DF57ObjectTag.class)
})
public abstract class ObjectTag<T1 extends SpcexObjectBase> {
public abstract class ObjectTag<T extends SpcexObjectBase> {
private final ETable prefix;
private final Class<T> clazz;
private final String nameOfMap;
protected HazelcastService hazelcastService;
protected ImdgHazelcast<T> map;
@JsonIgnore
protected String fileName;
@JsonIgnore
protected Long generationId;
public ObjectTag(ETable prefix) {
public ObjectTag(ETable prefix, Class<T> clazz, String nameOfMap) {
this.prefix = prefix;
this.clazz = clazz;
this.nameOfMap = nameOfMap;
}
public abstract T1 getSDfEntity();
public abstract T getSDfEntity();
public ETable getPrefix() {
return prefix;
@ -38,6 +45,10 @@ public abstract class ObjectTag<T1 extends SpcexObjectBase> {
return fileName;
}
public void setHazelcastService(HazelcastService hazelcastService) {
this.hazelcastService = hazelcastService;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
@ -50,14 +61,18 @@ public abstract class ObjectTag<T1 extends SpcexObjectBase> {
this.generationId = generationId;
}
public void insertEntity(ImdgHazelcast<T1> map) {
T1 obj = getSDfEntity();
if (checkOnExisting(obj)) {
map.insert(obj);
public void injectEntity(T obj) {
if (map == null) {
bootMap();
}
map.insert(obj);
}
protected boolean checkOnExisting(T1 obj) {
public boolean checkOnExisting(T obj) {
return true;
}
protected void bootMap() {
map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
}
}

29
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ImportToDB.java
View file

@ -1,36 +1,29 @@
package ru.spcex.clearing.xml.importer.logic.steps;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.clearing.classes.statics.data.sdf.SDf57;
import ru.spcex.clearing.xml.importer.logic.data.ResultContainer;
import ru.spcex.clearing.xml.importer.logic.data.enums.ETable;
import ru.spcex.clearing.xml.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.xml.importer.logic.data.tags.DocumentTag;
import ru.spcex.clearing.xml.importer.logic.data.tags.objects.ObjectTag;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
import ru.spcex.platform.enumeration.ObjectType;
import ru.spcex.platform.enumeration.Priority;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
@Component
public class ImportToDB {
private final Logger log = LoggerFactory.getLogger(getClass());
private final XmlMapper xmlMapper;
private final HazelcastService hazelcastService;
private final XmlImportKafkaMessenger kafkaMessenger;
private final Map<ETable, ImdgHazelcast<?>> mapOfTable;
public ImportToDB(@Qualifier("xmlMapper") XmlMapper xmlMapper,
HazelcastService hazelcastService,
XmlImportKafkaMessenger kafkaMessenger,
@Qualifier("mapOfTable") Map<ETable, ImdgHazelcast<?>> mapOfTable) {
this.xmlMapper = xmlMapper;
public ImportToDB(HazelcastService hazelcastService,
XmlImportKafkaMessenger kafkaMessenger) {
this.hazelcastService = hazelcastService;
this.kafkaMessenger = kafkaMessenger;
this.mapOfTable = mapOfTable;
}
public StageResult process(ResultContainer resultContainer) {
@ -41,10 +34,18 @@ public class ImportToDB {
String fileName = resultContainer.getXmlFile().getName();
Long fileId = hazelcastService.getImdgIdGenerator().nextId();
for (ObjectTag o : documentTag.getObjects()) {
ImdgHazelcast<? extends SpcexObjectBase> imdgMap = mapOfTable.get(o.getPrefix());
o.setHazelcastService(hazelcastService);
o.setFileName(fileName);
o.setGenerationId(fileId);
o.insertEntity(imdgMap);
SpcexObjectBase entityTable = o.getSDfEntity();
if (!o.checkOnExisting(entityTable) && entityTable instanceof SDf57) {
kafkaMessenger.sendUserNotification(ObjectType.rgst,
"Номер транзакции " + ((SDf57) entityTable).getDbfId() + " в полученном df57 уже был обработан ранее",
Priority.HIGH);
} else {
o.injectEntity(entityTable);
}
}
if (ETable.DF_01.equals(currTable)) {

5
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/XmlImportKafkaMessenger.java
View file

@ -1,5 +1,7 @@
package ru.spcex.clearing.xml.importer.logic.steps;
import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;
@ -9,7 +11,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.platform.messaging.domain.Consts;
import static ru.spcex.clearing.platform.messaging.domain.Consts.PAIR_SDF;
import ru.spcex.clearing.platform.messaging.domain.cud.balance.StatementRequest;
import ru.spcex.clearing.platform.messaging.domain.cud.clearing.Sdf04Request;
import ru.spcex.clearing.platform.messaging.domain.cud.utilities.NotificationNewRequest;
@ -76,7 +77,7 @@ public class XmlImportKafkaMessenger implements InitializingBean {
}
}
private void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
public void sendUserNotification(ObjectType objectType, String comment, Priority priority) {
final String destination = Consts.NOTIFICATION_NEW;
NotificationNewRequest request = new NotificationNewRequest();
request.setObjectType(objectType.getKey());

6
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java
View file

@ -59,7 +59,11 @@ public class XMLImporterService {
}
public void processTable(ETable specificTable) {
log.info("adding import task {}", specificTable == null ? "without specific table" : specificTable);
if (specificTable == null) {
log.trace("adding import task without specific table");
} else {
log.info("adding import task {}", specificTable);
}
fileChecker.checkAndLoadSFTP();
executorService.execute(() -> {

2
pom.xml
View file

@ -53,6 +53,8 @@
<folder_root_registry-service>${folder_root_clearing}/clearing-parent/registry-service</folder_root_registry-service>
<folder_root_scheduler-service>${folder_root_clearing}/clearing-parent/scheduler-service</folder_root_scheduler-service>
<folder_root_gateway-api>${folder_root_clearing}/clearing-parent/gateway-api</folder_root_gateway-api>
<folder_root_xml-exporter>${folder_root_clearing}/clearing-parent/xml-exporter</folder_root_xml-exporter>
<folder_root_xml-importer>${folder_root_clearing}/clearing-parent/xml-importer</folder_root_xml-importer>
<!-- IMDG -->
<external_libraries.hazelcast.version>3.12.4</external_libraries.hazelcast.version>
<external_libraries.slf4j.version>1.7.33</external_libraries.slf4j.version>

46
z-distr/pom.xml
View file

@ -568,6 +568,52 @@
</fileSets>
</configuration>
</execution>
<execution>
<id>copy-xml-exporter-bin</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<fileSets>
<fileSet>
<sourceFile>${folder_root_xml-exporter}/target/xml-exporter.jar</sourceFile>
<destinationFile>${folder.clearing.distr.bin}/xml-exporter.jar</destinationFile>
</fileSet>
<fileSet>
<sourceFile>${folder_root_xml-exporter}/target/xml-exporter.jar</sourceFile>
<destinationFile>${folder.clearing.distr.services}/xml-exporter/xml-exporter.jar</destinationFile>
</fileSet>
<fileSet>
<sourceFile>${folder_root_xml-exporter}/src/main/resources/application.properties</sourceFile>
<destinationFile>${folder.clearing.distr.settings}/xml-exporter/application.properties</destinationFile>
</fileSet>
</fileSets>
</configuration>
</execution>
<execution>
<id>copy-xml-importer-bin</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<fileSets>
<fileSet>
<sourceFile>${folder_root_xml-importer}/target/xml-importer.jar</sourceFile>
<destinationFile>${folder.clearing.distr.bin}/xml-importer.jar</destinationFile>
</fileSet>
<fileSet>
<sourceFile>${folder_root_xml-importer}/target/xml-importer.jar</sourceFile>
<destinationFile>${folder.clearing.distr.services}/xml-importer/xml-importer.jar</destinationFile>
</fileSet>
<fileSet>
<sourceFile>${folder_root_xml-importer}/src/main/resources/application.properties</sourceFile>
<destinationFile>${folder.clearing.distr.settings}/xml-importer/application.properties</destinationFile>
</fileSet>
</fileSets>
</configuration>
</execution>
</executions>
</plugin>
<plugin>

4
z-distr/src/main/resources/distr/sh/kill_all.sh
View file

@ -19,4 +19,6 @@ kill -9 $(ps -ef | grep java | grep scheduler-service.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep reports-service.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep registry-service.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep swt-importer.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep xml-exporter.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep xml-importer.jar | awk '{print $2}')

2
z-distr/src/main/resources/distr/sh/launch_all.sh
View file

@ -20,3 +20,5 @@ cd /opt/mfd/clearing/bin
/opt/mfd/clearing/bin/registry-service.sh
/opt/mfd/clearing/bin/swt-importer.sh
/opt/mfd/clearing/bin/gateway-api.sh
/opt/mfd/clearing/bin/xml-exporter.sh
/opt/mfd/clearing/bin/xml-importer.sh

9
z-distr/src/main/resources/distr/sh/xml-exporter.sh Normal file
View file

@ -0,0 +1,9 @@
#!/bin/bash
CLEARING_HOME=/opt/mfd/clearing/
cd $CLEARING_HOME/bin
CMD="java -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=7031 -jar xml-exporter.jar --spring.config.location=$CLEARING_HOME/settings/xml-exporter/"
$CMD >/dev/null 2>&1 &

9
z-distr/src/main/resources/distr/sh/xml-importer.sh Normal file
View file

@ -0,0 +1,9 @@
#!/bin/bash
CLEARING_HOME=/opt/mfd/clearing/
cd $CLEARING_HOME/bin
CMD="java -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=7041 -jar xml-importer.jar --spring.config.location=$CLEARING_HOME/settings/xml-importer/"
$CMD >/dev/null 2>&1 &