diff --git a/clearing-parent/dbf-importer/pom.xml b/clearing-parent/dbf-importer/pom.xml index f60b0b139..9f8772d3e 100644 --- a/clearing-parent/dbf-importer/pom.xml +++ b/clearing-parent/dbf-importer/pom.xml @@ -1,6 +1,6 @@ - 4.0.0 @@ -50,9 +50,30 @@ com.github.albfernandez javadbf + + ru.spcex.platform + platform-imdg-api + + + ru.spcex.platform + platform-imdg-api-hazelcast-impl + + + ru.spcex.clearing + classes + + + + src/main/resources + + application.properties + + false + + org.springframework.boot @@ -69,6 +90,7 @@ + \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java index d330fc660..bd1dc294b 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java @@ -27,6 +27,7 @@ public class DBFImporterConfig { private final AProperties properties; private final ApplicationContext context; + public DBFImporterConfig(@Qualifier("dbfImporterProperties") AProperties properties, ApplicationContext context) { this.properties = properties; this.context = context; diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/ImporterImdgConfig.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/ImporterImdgConfig.java new file mode 100644 index 000000000..04d48af97 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/ImporterImdgConfig.java @@ -0,0 +1,46 @@ +package ru.spcex.clearing.dbf.importer.config; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; + +@Configuration +public class ImporterImdgConfig { + private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) { + ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor(); + if (maxPoolSz > 2) { + pool.setKeepAliveSeconds(60); + pool.setAllowCoreThreadTimeOut(true); + } + pool.setCorePoolSize(maxPoolSz); + pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion); + return pool; + } + + @Bean(name = "taskExecutorHazelcastClientInitializer") + public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() { + return createThreadPoolTaskExecutor(1, true); + } + + @Bean(name = "taskExecutorIdGeneratorAwaiter") + public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() { + return createThreadPoolTaskExecutor(1, false); + } + + @Autowired + @Bean + public HazelcastService imdgProvider( + @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + ImportDBFServiceSettings settings + ) { + return new HazelcastService(taskExecutorHazelcastClientInitializer, + taskExecutorIdGeneratorAwaiter, + settings.getHazelcast()); + } + +} diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/settings/ImportDBFServiceSettings.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/settings/ImportDBFServiceSettings.java new file mode 100644 index 000000000..788715794 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/settings/ImportDBFServiceSettings.java @@ -0,0 +1,21 @@ +package ru.spcex.clearing.dbf.importer.config.settings; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; +import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams; + +@Component +@PropertySource("file:${spring.config.location}/application.properties") +@ConfigurationProperties("import-dbf-service") +public class ImportDBFServiceSettings { + private HazelcastClientParams hazelcast; + + public HazelcastClientParams getHazelcast() { + return hazelcast; + } + + public void setHazelcast(HazelcastClientParams hazelcast) { + this.hazelcast = hazelcast; + } +} diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java index 7f487ee9c..bb2f1128e 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java @@ -1,18 +1,18 @@ package ru.spcex.clearing.dbf.importer.logic.data; -import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import java.util.EnumMap; import java.util.Map; public class DBStructureDBF { - private Map tables = new EnumMap<>(Table.class); + private Map tables = new EnumMap<>(ETable.class); - public Map getTables() { + public Map getTables() { return tables; } - public void setTables(Map tables) { + public void setTables(Map tables) { this.tables = tables; } } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/ResultContainer.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/ResultContainer.java index 7a2471523..d022f674b 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/ResultContainer.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/ResultContainer.java @@ -1,6 +1,6 @@ package ru.spcex.clearing.dbf.importer.logic.data; -import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import java.io.File; import java.util.UUID; @@ -10,14 +10,15 @@ import java.util.UUID; */ public class ResultContainer { private UUID uuid; - private Table dbfTable; + private ETable dbfTable; private File dbfFile; private byte[] dbfSource; - protected ResultContainer() {} + protected ResultContainer() { + } - public static ResultContainer createNewTask(Table dbfTable, File dbfFile) { + public static ResultContainer createNewTask(ETable dbfTable, File dbfFile) { ResultContainer container = new ResultContainer(); container.dbfTable = dbfTable; container.dbfFile = dbfFile; @@ -25,11 +26,11 @@ public class ResultContainer { return container; } - public Table getDbfTable() { + public ETable getDbfTable() { return dbfTable; } - public void setDbfTable(Table dbfTable) { + public void setDbfTable(ETable dbfTable) { this.dbfTable = dbfTable; } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/ETable.java similarity index 58% rename from clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/Table.java rename to clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/ETable.java index d237910c7..4f4e962b5 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/Table.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/ETable.java @@ -1,35 +1,41 @@ package ru.spcex.clearing.dbf.importer.logic.data.enums; -public enum Table { +public enum ETable { DF_01("DF-01"), DF_02("DF-02"), DF_03("DF-03"), DF_04("DF-04"), DF_05("DF-05"), - DF_06("DF-06"); - + DF_08("DF-08"), + DF_09("DF-09"), + DF_10("DF-10"), + DF_11("DF-11"), + DF_12("DF-12"), + DF_16("DF-16"), + DF_17("DF-17"), + DF_18("DF-18"); private final String prefix; - Table(String prefix) { + ETable(String prefix) { this.prefix = prefix; } - public boolean fileForThisTable(String filename) { - return filename != null && filename.startsWith(prefix); - } - - public static Table getTableForFilename(String filename) { - for (Table table : Table.values()) { + public static ETable getTableForFilename(String filename) { + for (ETable table : ETable.values()) { if (table.fileForThisTable(filename)) return table; } return null; } - public static Table tableForName(String name) { - for (Table table : values()) { + public static ETable tableForName(String name) { + for (ETable table : values()) { if (name.equalsIgnoreCase(table.name())) return table; } return null; } + + public boolean fileForThisTable(String filename) { + return filename != null && filename.startsWith(prefix); + } } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java new file mode 100644 index 000000000..6fd6269a7 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java @@ -0,0 +1,73 @@ +package ru.spcex.clearing.dbf.importer.logic.data.tables; + +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.imdg.api.ImdgProvider; +import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; + +public abstract class AbstractTable { + protected HazelcastService hazelcastService; + protected ImdgHazelcast map; + private String prefix; + private Class clazz; + private String nameOfMap; + + protected AbstractTable(String prefix, Class clazz, String nameOfMap) { + this.prefix = prefix; + this.clazz = clazz; + this.nameOfMap = nameOfMap; + } + + public HazelcastService getHazelcastService() { + return hazelcastService; + } + + public void setHazelcastService(HazelcastService hazelcastService) { + this.hazelcastService = hazelcastService; + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + public Class getClazz() { + return clazz; + } + + public void setObject(Class clazz) { + this.clazz = clazz; + } + + public String getNameOfMap() { + return nameOfMap; + } + + public void setNameOfMap(String nameOfMap) { + this.nameOfMap = nameOfMap; + } + + public ImdgHazelcast getImdg() { + return map; + } + + public abstract T getEntity(Object[] entity); + + public void injectEntity(T obj) { + map.insert(obj); + } + + public ImdgHazelcast getMap() { + if (map == null) { + bootMap(); + } + return map; + } + + private void bootMap() { + map = (ImdgHazelcast) hazelcastService.getImdg(nameOfMap, clazz); + } +} diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java new file mode 100644 index 000000000..26974afe6 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java @@ -0,0 +1,37 @@ +package ru.spcex.clearing.dbf.importer.logic.data.tables; + +import ru.clearing.classes.statics.data.sdf.SDf01; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; + +public class SDf01Table extends AbstractTable { + + private static final String PREFIX = ETable.DF_01.name(); + private static final Class CLAZZ = SDf01.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf01; + + public SDf01Table() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + + @Override + public SDf01 getEntity(Object[] entity) { + SDf01 result = new SDf01(); + + result.setId(this.getMap().nextIDSequenceFor()); + result.setCurr_code((String) entity[0]); + result.setAccount((String) entity[1]); + result.setRemainder((String) entity[2]); + result.setDeal((String) entity[3]); + result.setAcc_code((String) entity[4]); + result.setDat((String) entity[5]); + result.setMarket((String) entity[6]); + result.setAcc_name((String) entity[7]); + result.setAcc_type((String) entity[8]); + result.setSumengage((String) entity[9]); + result.setSumunblock((String) entity[10]); + result.setFile_type((String) entity[11]); + return result; + } + +} \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java new file mode 100644 index 000000000..5822e1283 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java @@ -0,0 +1,43 @@ +package ru.spcex.clearing.dbf.importer.logic.data.tables; + +import ru.clearing.classes.statics.data.sdf.SDf02; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; +import ru.spcex.platform.utils.time.TimeUtil; + +public class SDf02Table extends AbstractTable { + + private static final String PREFIX = ETable.DF_02.name(); + private static final Class CLAZZ = SDf02.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf02; + + public SDf02Table() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + + @Override + public SDf02 getEntity(Object[] entity) { + SDf02 result = new SDf02(); + + result.setId(this.getMap().nextIDSequenceFor()); + result.setCurr_code((String) entity[0]); + result.setAccount((String) entity[1]); + result.setRemainder((String) entity[2]); + result.setDeal((String) entity[3]); + result.setAcc_code((String) entity[4]); + result.setDat((String) entity[5]); + result.setMarket((String) entity[6]); + result.setAcc_name((String) entity[7]); + result.setAcc_type((String) entity[8]); + result.setSumengage((String) entity[9]); + result.setSumunblock((String) entity[10]); + result.setFile_type((String) entity[11]); + result.setResult((String) entity[12]); + result.setGenerationTime(TimeUtil.strToInstant((String) entity[13])); + result.setGenerationId((Long) entity[14]); + result.setInSDf01Id((Long) entity[15]); + return result; + } + +} + diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java new file mode 100644 index 000000000..e2253df39 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java @@ -0,0 +1,33 @@ +package ru.spcex.clearing.dbf.importer.logic.data.tables; + +import ru.clearing.classes.statics.data.sdf.SDf08; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; +import ru.spcex.platform.utils.time.TimeUtil; + +import java.math.BigDecimal; + +public class SDf08Table extends AbstractTable { + + private static final String PREFIX = ETable.DF_08.name(); + private static final Class CLAZZ = SDf08.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf08; + + + public SDf08Table() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + + @Override + public SDf08 getEntity(Object[] entity) { + SDf08 result = new SDf08(); + result.setId(this.getMap().nextIDSequenceFor()); + result.setNumber((BigDecimal) entity[0]); + result.setDatetime(TimeUtil.strToInstant((String) entity[1])); + result.setGenerationTime(TimeUtil.strToInstant((String) entity[2])); + result.setGenerationId((Long) entity[3]); + return result; + } + + +} \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java new file mode 100644 index 000000000..df45fefea --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java @@ -0,0 +1,34 @@ +package ru.spcex.clearing.dbf.importer.logic.data.tables; + +import ru.clearing.classes.statics.data.sdf.SDf12; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; +import ru.spcex.platform.utils.time.TimeUtil; + +import java.math.BigDecimal; + +public class SDf12Table extends AbstractTable { + + private static final String PREFIX = ETable.DF_12.name(); + private static final Class CLAZZ = SDf12.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf12; + + public SDf12Table() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + + @Override + public SDf12 getEntity(Object[] entity) { + SDf12 result = new SDf12(); + + result.setId(this.getMap().nextIDSequenceFor()); + result.setAccount((String) entity[0]); + result.setDeal((String) entity[1]); + result.setStatus((BigDecimal) entity[2]); + result.setFileName((String) entity[3]); + result.setGenerationTime(TimeUtil.strToInstant((String) entity[4])); + result.setGenerationId((Long) entity[5]); + + return result; + } +} \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java new file mode 100644 index 000000000..b214f68af --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java @@ -0,0 +1,39 @@ +package ru.spcex.clearing.dbf.importer.logic.data.tables; + +import ru.clearing.classes.statics.data.sdf.SDf16; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; +import ru.spcex.platform.utils.time.TimeUtil; + +import java.math.BigDecimal; + +public class SDf16Table extends AbstractTable { + + private static final String PREFIX = ETable.DF_16.name(); + private static final Class CLAZZ = SDf16.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf16; + + public SDf16Table() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + + @Override + public SDf16 getEntity(Object[] entity) { + SDf16 result = new SDf16(); + + result.setId(this.getMap().nextIDSequenceFor()); + result.setDate(TimeUtil.strToInstant((String) entity[0])); + result.setAccount((String) entity[1]); + result.setSum((BigDecimal) entity[2]); + result.setMarket((String) entity[3]); + result.setType((String) entity[4]); + result.setINN((BigDecimal) entity[5]); + result.setBIC((BigDecimal) entity[6]); + result.setSPEC((String) entity[7]); + result.setNumber((BigDecimal) entity[8]); + result.setFileName((String) entity[9]); + result.setGenerationTime(TimeUtil.strToInstant((String) entity[10])); + result.setGenerationId((Long) entity[11]); + return result; + } +} \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java new file mode 100644 index 000000000..98a9d09c2 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java @@ -0,0 +1,33 @@ +package ru.spcex.clearing.dbf.importer.logic.data.tables; + +import ru.clearing.classes.statics.data.sdf.SDf18; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.imdg.IMDGDistributedNames; +import ru.spcex.platform.utils.time.TimeUtil; + +import java.math.BigDecimal; + +public class SDf18Table extends AbstractTable { + + private static final String PREFIX = ETable.DF_18.name(); + private static final Class CLAZZ = SDf18.class; + private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf18; + + public SDf18Table() { + super(PREFIX, CLAZZ, NAME_OF_HZ_MAP); + } + + @Override + public SDf18 getEntity(Object[] entity) { + SDf18 result = new SDf18(); + result.setId(this.getMap().nextIDSequenceFor()); + result.setAccount((String) entity[0]); + result.setDeal((String) entity[1]); + result.setStatus((BigDecimal) entity[2]); + result.setResult((BigDecimal) entity[3]); + result.setGenerationTime(TimeUtil.strToInstant((String) entity[4])); + result.setGenerationId((Long) entity[5]); + result.setInSDf12Id((Long) entity[6]); + return result; + } +} \ No newline at end of file diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java index b0d494996..413c1a6c9 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java @@ -1,28 +1,21 @@ package ru.spcex.clearing.dbf.importer.logic.stages; -import com.linuxense.javadbf.DBFField; import com.linuxense.javadbf.DBFReader; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.dao.DataAccessException; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; -import ru.spcex.clearing.dbf.importer.logic.data.enums.ColumnType; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult; -import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; +import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable; import ru.spcex.clearing.dbf.importer.properties.AProperties; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; + +import static ru.spcex.clearing.dbf.importer.utils.Const.MAPPING_ENUM_TABLE_OBJECT_TABLE; /** * Заливка проверенных данных в базу @@ -30,82 +23,32 @@ import java.util.Objects; @Component public class ImportToDB extends Stage { private final AProperties properties; - private final JdbcTemplate dbfJdbcTemplate; + private final HazelcastService hazelcastService; public ImportToDB(@Qualifier("dbfImporterProperties") AProperties properties, - @Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) { + HazelcastService hazelcastService) { this.properties = properties; - this.dbfJdbcTemplate = dbfJdbcTemplate; + this.hazelcastService = hazelcastService; + + } @Override public StageResult process(ResultContainer resultContainer) { - Objects.requireNonNull(resultContainer.getDbfTable()); - Objects.requireNonNull(resultContainer.getDbfSource()); - - Table currTable = resultContainer.getDbfTable(); + ETable currTable = resultContainer.getDbfTable(); byte[] source = resultContainer.getDbfSource(); Charset sourceCharset = Charset.forName(properties.getDbfEncoding()); try (InputStream is = new ByteArrayInputStream(source); DBFReader dbfReader = new DBFReader(is, sourceCharset)) { - int columnCount = dbfReader.getFieldCount(); - - List columnNames = new ArrayList<>(columnCount); - List columnTypes = new ArrayList<>(columnCount); - for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) { - DBFField currField = dbfReader.getField(columnIdx); - String fieldName = currField.getName(); - ColumnType columnType = ColumnType.getForDBFType(currField.getType()); - columnNames.add(fieldName); - columnTypes.add(columnType.getSqlType()); + AbstractTable table = MAPPING_ENUM_TABLE_OBJECT_TABLE.get(currTable); + table.setHazelcastService(hazelcastService); + for (int i = 0; i < dbfReader.getRecordCount(); i++) { + Object[] entity = dbfReader.nextRecord(); + table.injectEntity(table.getEntity(entity)); } - - int recordCount = dbfReader.getRecordCount(); - List values = new ArrayList<>(recordCount); - for (int rowIdx = 0; rowIdx < recordCount; rowIdx++) { - Object[] dbfRow = dbfReader.nextRecord(); - values.add(dbfRow); - } - - int maxBatchSize = properties.getInsertBatchSize(); - int batchCount = recordCount / maxBatchSize; - int batchRem = recordCount % maxBatchSize; - for (int batchIdx = 0; batchIdx < batchCount + 1; batchIdx++) { - if (batchIdx == batchCount && batchRem == 0) continue; - final Integer currentBatchIdx = batchIdx; - - int[] rowsInserted = dbfJdbcTemplate.batchUpdate( - "insert into " + currTable.name() + - " (" + String.join(",", columnNames) + ")" + - " values (" + String.join(",", Collections.nCopies(columnCount, "?")) + ")", - new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - Object[] currValuesRow = values.get(currentBatchIdx * maxBatchSize + i); - int parameterIndex = 1; - for (Object object : currValuesRow) { - ps.setObject(parameterIndex, object, columnTypes.get(parameterIndex - 1)); - parameterIndex++; - } - } - - @Override - public int getBatchSize() { - return currentBatchIdx == batchCount ? batchRem : maxBatchSize; - } - }); - log.info("uuid {}. Imported records: {}", resultContainer.getUuid(), rowsInserted.length); - } - - } catch (IOException e) { - log.error(String.format("uuid %s. Can't read source file %s.", - resultContainer.getUuid(), - resultContainer.getDbfFile().getName()), e); - return StageResult.ERROR; - } catch (DataAccessException e) { - log.error(String.format("uuid %s. Can't insert records", resultContainer.getUuid()), e); - return StageResult.ERROR; + } catch (IOException exception) { + log.warn(exception.getMessage()); } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java index 60807b36e..eb0aa7026 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java @@ -1,194 +1,39 @@ package ru.spcex.clearing.dbf.importer.logic.stages; -import com.linuxense.javadbf.DBFDataType; -import com.linuxense.javadbf.DBFField; -import com.linuxense.javadbf.DBFReader; import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.support.JdbcUtils; -import org.springframework.jdbc.support.MetaDataAccessException; import org.springframework.stereotype.Component; import ru.spcex.clearing.dbf.importer.exceptions.ConfigException; -import ru.spcex.clearing.dbf.importer.logic.data.ColumnStructureDBF; -import ru.spcex.clearing.dbf.importer.logic.data.DBStructureDBF; import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; -import ru.spcex.clearing.dbf.importer.logic.data.TableStructureDBF; -import ru.spcex.clearing.dbf.importer.logic.data.enums.ColumnType; import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult; -import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; import ru.spcex.clearing.dbf.importer.properties.AProperties; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.Charset; -import java.sql.ResultSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.UUID; /** * Фильтрация источника на предмет соответствия полей */ @Component public class ValidateFields extends Stage implements InitializingBean { - private final JdbcTemplate dbfJdbcTemplate; private final AProperties properties; - - private DBStructureDBF dbStructureDBF; private Charset dbfCharset; - public ValidateFields(@Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate, AProperties properties) { - this.dbfJdbcTemplate = dbfJdbcTemplate; + public ValidateFields(AProperties properties) { this.properties = properties; } @Override public StageResult process(ResultContainer resultContainer) { - Objects.requireNonNull(resultContainer.getDbfTable()); - Objects.requireNonNull(resultContainer.getDbfSource()); - UUID taskUuid = resultContainer.getUuid(); - Table currTable = resultContainer.getDbfTable(); - byte[] source = resultContainer.getDbfSource(); - - TableStructureDBF currTableStructure = dbStructureDBF.getTables().get(currTable); - if (currTableStructure == null) { - log.error("uuid {}. Unknown table {}.", taskUuid, currTable); - return StageResult.ERROR; - } - - log.info("uuid {}. Check source {}", taskUuid, currTable); - - try (InputStream is = new ByteArrayInputStream(source); - DBFReader dbfReader = new DBFReader(is, dbfCharset)) { - - int columnCount = dbfReader.getFieldCount(); - if (columnCount != currTableStructure.getColumns().size()) { - log.error("uuid {}. Source for table {} doesn't match DB columns count. Source column count: {}. DB column count: {}.", - taskUuid, - currTable, - columnCount, - currTableStructure.getColumns().size()); - return StageResult.ERROR; - } - - boolean valid = true; - for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) { - DBFField currField = dbfReader.getField(columnIdx); - - String currFieldName = currField.getName(); - ColumnStructureDBF currColumnStructure = currTableStructure.getIgnoreCase(currFieldName); - if (currColumnStructure == null) { - log.error("uuid {}. Source for table {} contain unknown field {}. ", - taskUuid, - currTable, - currFieldName); - valid = false; - continue; - } - - DBFDataType dbfDataType = currField.getType(); - if (!currColumnStructure.getType().getDbfType().equals(dbfDataType)) { - log.error("uuid {}. Field {} from source {} has wrong data type (actual {}, expected {}).", - taskUuid, - currFieldName, - currTable, - dbfDataType != null ? dbfDataType.name() : "null", - currColumnStructure.getType()); - valid = false; - continue; - } - - int fieldLength = currField.getLength(); - if (fieldLength != currColumnStructure.getLength()) { - - if (!dbfDataType.equals(DBFDataType.DATE)) { - log.error("uuid {}. Field {} from source {} has wrong length (actual {}, expected {}).", - taskUuid, - currFieldName, - currTable, - fieldLength, - currColumnStructure.getLength()); - valid = false; - continue; - } else { - if (fieldLength > currColumnStructure.getLength()) { - log.error("uuid {}. Field {} with DATE type has wrong length (length from file > length db field)", - taskUuid, - currFieldName); - valid = false; - continue; - } - } - } - - int decimalCount = currField.getDecimalCount(); - if (decimalCount != currColumnStructure.getDecimalDigits()) { - log.error("uuid {}. Field {} from source {} has wrong decimal count (actual {}, expected {}).", - taskUuid, - currFieldName, - currTable, - decimalCount, - currColumnStructure.getDecimalDigits()); - valid = false; - } - - if (!valid) { - return StageResult.ERROR; - } - } - } catch (IOException e) { - log.error(String.format("uuid %s. Can't read source for table %s.", taskUuid, currTable), e); - return StageResult.ERROR; - } - return StageResult.OK; } - /** - * 1. Собирает из БД названия столбцов для дальнейшей валидации - * 2. Проверяет кодировку из настройки dbf.encoding-source + * 1. Проверяет кодировку из настройки dbf.encoding-source */ @Override public void afterPropertiesSet() throws Exception { - initDBStructure(); initDBFCharset(); } - private void initDBStructure() throws MetaDataAccessException { - dbStructureDBF = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()), - dbMeta -> { - DBStructureDBF dbStructureDBF = new DBStructureDBF(); - List tableNamesFromResultSet = new LinkedList<>(); - ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"}); - while (tableNamesRS.next()) { - tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME")); - } - for (String tableNameFromResultSet : tableNamesFromResultSet) { - Table currTable = Table.tableForName(tableNameFromResultSet); - if (currTable == null) continue; - TableStructureDBF tableStructureDBF = new TableStructureDBF(); - ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null); - while (columnNamesRS.next()) { - ColumnStructureDBF columnStructureDBF = new ColumnStructureDBF(); - columnStructureDBF.setName(columnNamesRS.getString("COLUMN_NAME")); - columnStructureDBF.setType(ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE"))); - columnStructureDBF.setLength(columnNamesRS.getInt("COLUMN_SIZE")); - columnStructureDBF.setDecimalDigits(columnNamesRS.getInt("DECIMAL_DIGITS")); - columnStructureDBF.setComment(columnNamesRS.getString("REMARKS")); - tableStructureDBF.getColumns().put(columnStructureDBF.getName(), columnStructureDBF); - } - assert !dbStructureDBF.getTables().containsKey(currTable); - - dbStructureDBF.getTables().put(currTable, tableStructureDBF); - } - return dbStructureDBF; - }); - } - private void initDBFCharset() { try { dbfCharset = Charset.forName(properties.getDbfEncoding()); @@ -196,5 +41,4 @@ public class ValidateFields extends Stage implements InitializingBean { throw new ConfigException("Unknown encoding from properties: " + properties.getDbfEncoding(), e); } } - } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java index 8d3ea5c70..92ab1e2d9 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java @@ -6,7 +6,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; -import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import ru.spcex.clearing.dbf.importer.logic.stages.Processor; import java.io.File; @@ -30,9 +30,9 @@ public class DBFImporterService { @Scheduled(cron = "${dbf.check-src-dir-cron}") public void run() { - Map> newFiles = fileChecker.checkNewFiles(); - for (Map.Entry> newFilesEntry : newFiles.entrySet()) { - Table currTable = newFilesEntry.getKey(); + Map> newFiles = fileChecker.checkNewFiles(); + for (Map.Entry> newFilesEntry : newFiles.entrySet()) { + ETable currTable = newFilesEntry.getKey(); List fileList = newFilesEntry.getValue(); for (File dbfFile : fileList) { executorService.execute(() -> processor.process(ResultContainer.createNewTask(currTable, dbfFile))); diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java index c7e7ddd9c..a468e4d80 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java @@ -1,7 +1,7 @@ package ru.spcex.clearing.dbf.importer.services; import org.springframework.stereotype.Service; -import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; import ru.spcex.clearing.dbf.importer.properties.AProperties; import java.io.File; @@ -15,8 +15,8 @@ public class FileChecker { this.properties = properties; } - public Map> checkNewFiles() { - Map> newFiles = new EnumMap<>(Table.class); + public Map> checkNewFiles() { + Map> newFiles = new EnumMap<>(ETable.class); String srcDir = properties.getSrcDir(); List dbfFiles = lsDBF(srcDir); @@ -25,7 +25,7 @@ public class FileChecker { for (File dbfFile : dbfFiles) { if (dbfFile.isDirectory()) continue; - Table currTable = Table.getTableForFilename(dbfFile.getName()); + ETable currTable = ETable.getTableForFilename(dbfFile.getName()); if (currTable == null) continue; List currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>()); diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/utils/Const.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/utils/Const.java new file mode 100644 index 000000000..eab95cd12 --- /dev/null +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/utils/Const.java @@ -0,0 +1,23 @@ +package ru.spcex.clearing.dbf.importer.utils; + +import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable; +import ru.spcex.clearing.dbf.importer.logic.data.tables.*; + +import java.util.HashMap; +import java.util.Map; + +public class Const { + + public static final Map MAPPING_ENUM_TABLE_OBJECT_TABLE = new HashMap<>(); + + static { + MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_01, new SDf01Table()); + MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_02, new SDf02Table()); + MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_08, new SDf08Table()); + MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_12, new SDf12Table()); + MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_16, new SDf16Table()); + MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_18, new SDf18Table()); + } + + +} diff --git a/clearing-parent/dbf-importer/src/main/resources/application.properties b/clearing-parent/dbf-importer/src/main/resources/application.properties index 451d47ee2..390c6f0e7 100644 --- a/clearing-parent/dbf-importer/src/main/resources/application.properties +++ b/clearing-parent/dbf-importer/src/main/resources/application.properties @@ -1,17 +1,16 @@ server.port=8080 server.servlet.context-path=/importer spring.main.web-application-type=servlet - db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres db.driver=org.postgresql.Driver db.login=clearing db.password=Aa111111 - dbf.check-src-dir-cron=* * * * 1 ? dbf.encoding-source=cp866 dbf.insert-batch-size=100 - dbf.delete-src-files=false dbf.src-dir=D:\\dbf\\ - -dbf.threads-count=10 \ No newline at end of file +dbf.threads-count=10 +import-dbf-service.hazelcast.cluster-members=127.0.0.1 +import-dbf-service.hazelcast.login=dev +import-dbf-service.hazelcast.password=dev-pass \ No newline at end of file diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java index c8c908449..cb848122f 100644 --- a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java @@ -13,6 +13,7 @@ import ru.spcex.platform.imdg.api.Imdg; import ru.spcex.platform.utils.log.ExceptionUtils; import java.util.*; + public class ImdgHazelcast implements Imdg { private final Logger log = LoggerFactory.getLogger(getClass()); private IdGenerator idGenerator; @@ -109,7 +110,7 @@ public class ImdgHazelcast implements Imdg { Set ids = map.keySet(or); Iterator idIterator = ids.iterator(); Collection searchResult = new ArrayList<>(); - while(idIterator.hasNext()) { + while (idIterator.hasNext()) { T element = map.get(idIterator.next()); if (element != null) { searchResult.add(element); @@ -118,6 +119,11 @@ public class ImdgHazelcast implements Imdg { return searchResult; } + @Override + public Long nextIDSequenceFor() { + return idGenerator.newId(); + } + public IMap getMap() { return map; } diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgTransactionalHazelcast.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgTransactionalHazelcast.java index 0aca655c3..5f19ae2e6 100644 --- a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgTransactionalHazelcast.java +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgTransactionalHazelcast.java @@ -105,7 +105,7 @@ public class ImdgTransactionalHazelcast implements Im Set ids = map.keySet(or); Iterator idIterator = ids.iterator(); Collection searchResult = new ArrayList<>(); - while(idIterator.hasNext()) { + while (idIterator.hasNext()) { T element = map.get(idIterator.next()); if (element != null) { searchResult.add(element); @@ -145,4 +145,9 @@ public class ImdgTransactionalHazelcast implements Im public void setMapName(String mapName) { this.mapName = mapName; } + + @Override + public Long nextIDSequenceFor() { + return idGenerator.newId(); + } } diff --git a/platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/time/TimeUtil.java b/platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/time/TimeUtil.java index a45d2564f..3b9d7b8b7 100644 --- a/platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/time/TimeUtil.java +++ b/platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/time/TimeUtil.java @@ -4,8 +4,10 @@ import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.Objects; public class TimeUtil { + public static final DateTimeFormatter PROPERTY_DATE_FORMATTER = DateTimeFormatter.ofPattern("dd.MM.yyyy"); public static ZoneId zone = ZoneId.systemDefault(); /** @@ -27,4 +29,17 @@ public class TimeUtil { public static Instant localDateToInstant(LocalDate date) { return date.atStartOfDay(zone).toInstant(); } + + public static LocalDate strToLocalDate(String date) { + try { + LocalDate parsed = LocalDate.parse(date, PROPERTY_DATE_FORMATTER); + return parsed; + } catch (Throwable e) { + return null; + } + } + + public static Instant strToInstant(String date) { + return localDateToInstant(Objects.requireNonNull(strToLocalDate(date))); + } }