From 5ad4034388a6c6cf4994b72d70fd428997ad63a6 Mon Sep 17 00:00:00 2001 From: aalehin Date: Thu, 22 Sep 2022 19:37:24 +0300 Subject: [PATCH] =?UTF-8?q?1)=20=D0=BF=D0=B5=D1=80=D0=B5=D0=BC=D0=B5=D1=89?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B4=D0=B0=D0=BD=D0=BD=D1=8B=D1=85?= =?UTF-8?q?=20=D0=B8=D0=B7=20=D0=BE=D0=B4=D0=BD=D0=BE=D0=B9=20=D0=B4=D0=B8?= =?UTF-8?q?=D1=80=D0=B5=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=B8=20=D0=B2=20?= =?UTF-8?q?=D0=B4=D1=80=D1=83=D0=B3=D1=83=D1=8E=202)=20=D0=B2=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=BB=D0=BD=D0=B8=D0=BB=20=D0=B7=D0=B0=D0=BF=D0=BE=D0=BB?= =?UTF-8?q?=D0=BD=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BF=D0=BE=D0=BB=D0=B5=D0=B9?= =?UTF-8?q?=20generation=5Fid,=20generation=5Ftime,=20file=5Fname?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../importer/config/DBFImporterConfig.java | 6 +- .../logic/data/tables/AbstractTable.java | 5 ++ .../logic/data/tables/SDf01Table.java | 5 ++ .../logic/data/tables/SDf02Table.java | 5 ++ .../logic/data/tables/SDf08Table.java | 6 +- .../logic/data/tables/SDf12Table.java | 9 ++- .../logic/data/tables/SDf16Table.java | 7 +- .../logic/data/tables/SDf18Table.java | 9 +-- .../logic/stages/ChangeDirOfFileStage.java | 65 +++++++++++++++++++ .../dbf/importer/logic/stages/ImportToDB.java | 3 +- .../dbf/importer/properties/AProperties.java | 10 +-- .../src/main/resources/application.properties | 1 + 12 files changed, 108 insertions(+), 23 deletions(-) create mode 100644 clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ChangeDirOfFileStage.java diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java index 52c59413c..43ca6d69a 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java @@ -7,10 +7,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import ru.spcex.clearing.dbf.importer.logic.stages.ImportToDB; -import ru.spcex.clearing.dbf.importer.logic.stages.LoadFileFromDisk; -import ru.spcex.clearing.dbf.importer.logic.stages.Stage; -import ru.spcex.clearing.dbf.importer.logic.stages.ValidateFields; +import ru.spcex.clearing.dbf.importer.logic.stages.*; import ru.spcex.clearing.dbf.importer.properties.AProperties; import java.util.LinkedList; @@ -36,6 +33,7 @@ public class DBFImporterConfig { pipeline.add(context.getBean(LoadFileFromDisk.class)); pipeline.add(context.getBean(ValidateFields.class)); pipeline.add(context.getBean(ImportToDB.class)); + pipeline.add(context.getBean(ChangeDirOfFileStage.class)); return pipeline; } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java index d961ae4b6..97fd9a495 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java @@ -10,6 +10,7 @@ public abstract class AbstractTable { private final String nameOfMap; protected HazelcastService hazelcastService; protected ImdgHazelcast map; + protected String filename; protected AbstractTable(String prefix, Class clazz, String nameOfMap) { this.prefix = prefix; @@ -17,6 +18,10 @@ public abstract class AbstractTable { this.nameOfMap = nameOfMap; } + public void setFilename(String filename) { + this.filename = filename; + } + public void setHazelcastService(HazelcastService hazelcastService) { this.hazelcastService = hazelcastService; } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java index 159af343b..0f5738ede 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java @@ -4,6 +4,8 @@ import ru.clearing.classes.statics.data.sdf.SDf01; import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import ru.spcex.clearing.imdg.IMDGDistributedNames; +import java.time.Instant; + public class SDf01Table extends AbstractTable { private static final String PREFIX = ETable.DF_01.name(); @@ -30,6 +32,9 @@ public class SDf01Table extends AbstractTable { result.setSumengage((String) entity[9]); result.setSumunblock((String) entity[10]); result.setFile_type((String) entity[11]); + result.setFileName(filename); + result.setGenerationTime(Instant.now()); + result.setGenerationId(0L); return result; } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java index 0be2113bb..9ddb14680 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java @@ -5,6 +5,8 @@ import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.platform.utils.time.TimeUtil; +import java.time.Instant; + public class SDf02Table extends AbstractTable { private static final String PREFIX = ETable.DF_02.name(); @@ -35,6 +37,9 @@ public class SDf02Table extends AbstractTable { result.setGenerationTime(TimeUtil.strToInstant((String) entity[13])); result.setGenerationId((Long) entity[14]); result.setInSDf01Id((Long) entity[15]); + //result.setFileName(filename); + result.setGenerationTime(Instant.now()); + result.setGenerationId(0L); return result; } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java index 58ba75200..72a3ac169 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java @@ -6,6 +6,7 @@ import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.platform.utils.time.TimeUtil; import java.math.BigDecimal; +import java.time.Instant; public class SDf08Table extends AbstractTable { @@ -24,8 +25,9 @@ public class SDf08Table extends AbstractTable { result.setNumber((BigDecimal) entity[0]); result.setDatetime(TimeUtil.strToInstant((String) entity[1])); - result.setGenerationTime(TimeUtil.strToInstant((String) entity[2])); - result.setGenerationId((Long) entity[3]); + //result.setFileName(filename); + result.setGenerationTime(Instant.now()); + result.setGenerationId(0L); return result; } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java index 4c8265d74..81c4a74a4 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java @@ -3,9 +3,9 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables; import ru.clearing.classes.statics.data.sdf.SDf12; import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import ru.spcex.clearing.imdg.IMDGDistributedNames; -import ru.spcex.platform.utils.time.TimeUtil; import java.math.BigDecimal; +import java.time.Instant; public class SDf12Table extends AbstractTable { @@ -25,10 +25,9 @@ public class SDf12Table extends AbstractTable { result.setAccount((String) entity[0]); result.setDeal((String) entity[1]); result.setStatus((BigDecimal) entity[2]); - result.setFileName((String) entity[3]); - result.setGenerationTime(TimeUtil.strToInstant((String) entity[4])); - result.setGenerationId((Long) entity[5]); - + result.setFileName(filename); + result.setGenerationTime(Instant.now()); + result.setGenerationId(0L); return result; } } \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java index e7dd034ba..611560b28 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java @@ -6,6 +6,7 @@ import ru.spcex.clearing.imdg.IMDGDistributedNames; import ru.spcex.platform.utils.time.TimeUtil; import java.math.BigDecimal; +import java.time.Instant; public class SDf16Table extends AbstractTable { @@ -31,9 +32,9 @@ public class SDf16Table extends AbstractTable { result.setBIC((BigDecimal) entity[6]); result.setSPEC((String) entity[7]); result.setNumber((BigDecimal) entity[8]); - result.setFileName((String) entity[9]); - result.setGenerationTime(TimeUtil.strToInstant((String) entity[10])); - result.setGenerationId((Long) entity[11]); + result.setFileName(filename); + result.setGenerationTime(Instant.now()); + result.setGenerationId(0L); return result; } } \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java index 478910708..ae5a35831 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java @@ -3,9 +3,9 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables; import ru.clearing.classes.statics.data.sdf.SDf18; import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import ru.spcex.clearing.imdg.IMDGDistributedNames; -import ru.spcex.platform.utils.time.TimeUtil; import java.math.BigDecimal; +import java.time.Instant; public class SDf18Table extends AbstractTable { @@ -20,13 +20,14 @@ public class SDf18Table extends AbstractTable { @Override public SDf18 getEntity(Object[] entity) { SDf18 result = new SDf18(); - + result.setAccount((String) entity[0]); result.setDeal((String) entity[1]); result.setStatus((BigDecimal) entity[2]); result.setResult((BigDecimal) entity[3]); - result.setGenerationTime(TimeUtil.strToInstant((String) entity[4])); - result.setGenerationId((Long) entity[5]); + // result.setFileName(filename); + result.setGenerationTime(Instant.now()); + result.setGenerationId(0L); result.setInSDf12Id((Long) entity[6]); return result; } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ChangeDirOfFileStage.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ChangeDirOfFileStage.java new file mode 100644 index 000000000..96bc33fbb --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ChangeDirOfFileStage.java @@ -0,0 +1,65 @@ +package ru.spcex.clearing.dbf.importer.logic.stages; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.importer.properties.AProperties; +import ru.spcex.platform.utils.time.TimeUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.time.Instant; +import java.time.format.DateTimeFormatter; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + + +@Component +public class ChangeDirOfFileStage extends Stage { + + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy.MM.dd HH.mm.ss"); + private final AProperties properties; + + @Autowired + public ChangeDirOfFileStage(AProperties properties) { + this.properties = properties; + } + + + @Override + public StageResult process(ResultContainer resultContainer) { + File srcDir = new File(properties.getSrcDir()); + File outDir = new File(properties.getOutDir()); + File dbfFile = resultContainer.getDbfFile(); + + if (!srcDir.exists()) { + log.error("SettlementHouse_DocIn does not exists!"); + return StageResult.ERROR; + } + if (!outDir.exists()) { + log.warn("SettlementHouse_DocIn does not exists! Trying to made new!"); + if (outDir.mkdir()) { + log.error("Made dir {} successfully!", outDir.getName()); + } else { + log.error("SettlementHouse_DocIn does not exists!"); + return StageResult.ERROR; + } + } + if (!dbfFile.exists()) { + log.error("DBF file to save does not exists!"); + return StageResult.ERROR; + } + String newNameOfFile = TimeUtil.formatInstant(Instant.now(), FORMATTER) + '_' + dbfFile.getName(); + + try { + Files.move(dbfFile.toPath(), outDir.toPath().resolve(newNameOfFile), REPLACE_EXISTING); + } catch (IOException e) { + log.error("Could not move file {}", e.getMessage()); + return StageResult.ERROR; + } + + return StageResult.COMPLETE; + } +} diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java index b4a190d5a..59a315a75 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java @@ -43,6 +43,7 @@ public class ImportToDB extends Stage { DBFReader dbfReader = new DBFReader(is, sourceCharset)) { AbstractTable table = MAPPING_ENUM_TABLE_OBJECT_TABLE.get(currTable); table.setHazelcastService(hazelcastService); + table.setFilename(resultContainer.getDbfFile().getName()); for (int i = 0; i < dbfReader.getRecordCount(); i++) { Object[] entity = dbfReader.nextRecord(); table.injectEntity(table.getEntity(entity)); @@ -53,6 +54,6 @@ public class ImportToDB extends Stage { } - return StageResult.COMPLETE; + return StageResult.OK; } } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/properties/AProperties.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/properties/AProperties.java index f34697e96..bc7d774e7 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/properties/AProperties.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/properties/AProperties.java @@ -8,19 +8,21 @@ public class AProperties { @Value("${dbf.src-dir}") private String srcDir; - + @Value("${dbf.out-dir}") + private String outDir; @Value("${dbf.delete-src-files}") private boolean deleteSrcFiles = true; - @Value("${dbf.encoding-source}") private String dbfEncoding; - @Value("${dbf.insert-batch-size}") private int insertBatchSize; - @Value("${dbf.threads-count}") private int threadsCount; + public String getOutDir() { + return outDir; + } + public String getSrcDir() { return srcDir; } diff --git a/clearing-parent/dbf-importer/src/main/resources/application.properties b/clearing-parent/dbf-importer/src/main/resources/application.properties index 390c6f0e7..3316a6c6e 100644 --- a/clearing-parent/dbf-importer/src/main/resources/application.properties +++ b/clearing-parent/dbf-importer/src/main/resources/application.properties @@ -10,6 +10,7 @@ dbf.encoding-source=cp866 dbf.insert-batch-size=100 dbf.delete-src-files=false dbf.src-dir=D:\\dbf\\ +dbf.out-dir=D:\\dbf\\out dbf.threads-count=10 import-dbf-service.hazelcast.cluster-members=127.0.0.1 import-dbf-service.hazelcast.login=dev