importer almost done

clearing-parent/dbf-importer/pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
@@ -50,9 +50,30 @@
             <groupId>com.github.albfernandez</groupId>
             <artifactId>javadbf</artifactId>
         </dependency>
+        <dependency>
+            <groupId>ru.spcex.platform</groupId>
+            <artifactId>platform-imdg-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.platform</groupId>
+            <artifactId>platform-imdg-api-hazelcast-impl</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.clearing</groupId>
+            <artifactId>classes</artifactId>
+        </dependency>
     </dependencies>
 
     <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <excludes>
+                    <exclude>application.properties</exclude>
+                </excludes>
+                <filtering>false</filtering>
+            </resource>
+        </resources>
         <plugins>
             <plugin>
                 <groupId>org.springframework.boot</groupId>
@@ -69,6 +90,7 @@
                 </configuration>
             </plugin>
         </plugins>
+
     </build>
 
 </project>

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java

@@ -27,6 +27,7 @@ public class DBFImporterConfig {
     private final AProperties properties;
     private final ApplicationContext context;
 
+
     public DBFImporterConfig(@Qualifier("dbfImporterProperties") AProperties properties, ApplicationContext context) {
         this.properties = properties;
         this.context = context;

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/ImporterImdgConfig.java

@@ -0,0 +1,46 @@
+package ru.spcex.clearing.dbf.importer.config;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings;
+import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
+
+@Configuration
+public class ImporterImdgConfig {
+    private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) {
+        ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
+        if (maxPoolSz > 2) {
+            pool.setKeepAliveSeconds(60);
+            pool.setAllowCoreThreadTimeOut(true);
+        }
+        pool.setCorePoolSize(maxPoolSz);
+        pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion);
+        return pool;
+    }
+
+    @Bean(name = "taskExecutorHazelcastClientInitializer")
+    public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() {
+        return createThreadPoolTaskExecutor(1, true);
+    }
+
+    @Bean(name = "taskExecutorIdGeneratorAwaiter")
+    public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() {
+        return createThreadPoolTaskExecutor(1, false);
+    }
+
+    @Autowired
+    @Bean
+    public HazelcastService imdgProvider(
+            @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
+            @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
+            ImportDBFServiceSettings settings
+    ) {
+        return new HazelcastService(taskExecutorHazelcastClientInitializer,
+                taskExecutorIdGeneratorAwaiter,
+                settings.getHazelcast());
+    }
+
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/settings/ImportDBFServiceSettings.java

@@ -0,0 +1,21 @@
+package ru.spcex.clearing.dbf.importer.config.settings;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
+import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams;
+
+@Component
+@PropertySource("file:${spring.config.location}/application.properties")
+@ConfigurationProperties("import-dbf-service")
+public class ImportDBFServiceSettings {
+    private HazelcastClientParams hazelcast;
+
+    public HazelcastClientParams getHazelcast() {
+        return hazelcast;
+    }
+
+    public void setHazelcast(HazelcastClientParams hazelcast) {
+        this.hazelcast = hazelcast;
+    }
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java

@@ -1,18 +1,18 @@
 package ru.spcex.clearing.dbf.importer.logic.data;
 
-import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
 
 import java.util.EnumMap;
 import java.util.Map;
 
 public class DBStructureDBF {
-    private Map<Table, TableStructureDBF> tables = new EnumMap<>(Table.class);
+    private Map<ETable, TableStructureDBF> tables = new EnumMap<>(ETable.class);
 
-    public Map<Table, TableStructureDBF> getTables() {
+    public Map<ETable, TableStructureDBF> getTables() {
         return tables;
     }
 
-    public void setTables(Map<Table, TableStructureDBF> tables) {
+    public void setTables(Map<ETable, TableStructureDBF> tables) {
         this.tables = tables;
     }
 }

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/ResultContainer.java

@@ -1,6 +1,6 @@
 package ru.spcex.clearing.dbf.importer.logic.data;
 
-import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
 
 import java.io.File;
 import java.util.UUID;
@@ -10,14 +10,15 @@ import java.util.UUID;
  */
 public class ResultContainer {
     private UUID uuid;
-    private Table dbfTable;
+    private ETable dbfTable;
     private File dbfFile;
 
     private byte[] dbfSource;
 
-    protected ResultContainer() {}
+    protected ResultContainer() {
+    }
 
-    public static ResultContainer createNewTask(Table dbfTable, File dbfFile) {
+    public static ResultContainer createNewTask(ETable dbfTable, File dbfFile) {
         ResultContainer container = new ResultContainer();
         container.dbfTable = dbfTable;
         container.dbfFile = dbfFile;
@@ -25,11 +26,11 @@ public class ResultContainer {
         return container;
     }
 
-    public Table getDbfTable() {
+    public ETable getDbfTable() {
         return dbfTable;
     }
 
-    public void setDbfTable(Table dbfTable) {
+    public void setDbfTable(ETable dbfTable) {
         this.dbfTable = dbfTable;
     }
 

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/ETable.java

@@ -1,35 +1,41 @@
 package ru.spcex.clearing.dbf.importer.logic.data.enums;
 
-public enum Table {
+public enum ETable {
     DF_01("DF-01"),
     DF_02("DF-02"),
     DF_03("DF-03"),
     DF_04("DF-04"),
     DF_05("DF-05"),
-    DF_06("DF-06");
+    DF_08("DF-08"),
-
+    DF_09("DF-09"),
+    DF_10("DF-10"),
+    DF_11("DF-11"),
+    DF_12("DF-12"),
+    DF_16("DF-16"),
+    DF_17("DF-17"),
+    DF_18("DF-18");
     private final String prefix;
 
-    Table(String prefix) {
+    ETable(String prefix) {
         this.prefix = prefix;
     }
 
-    public boolean fileForThisTable(String filename) {
+    public static ETable getTableForFilename(String filename) {
-        return filename != null && filename.startsWith(prefix);
+        for (ETable table : ETable.values()) {
-    }
-
-    public static Table getTableForFilename(String filename) {
-        for (Table table : Table.values()) {
             if (table.fileForThisTable(filename)) return table;
         }
         return null;
     }
 
-    public static Table tableForName(String name) {
+    public static ETable tableForName(String name) {
-        for (Table table : values()) {
+        for (ETable table : values()) {
             if (name.equalsIgnoreCase(table.name()))
                 return table;
         }
         return null;
     }
+
+    public boolean fileForThisTable(String filename) {
+        return filename != null && filename.startsWith(prefix);
+    }
 }

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java

@@ -0,0 +1,73 @@
+package ru.spcex.clearing.dbf.importer.logic.data.tables;
+
+import ru.spcex.platform.classes.base.SpcexObjectBase;
+import ru.spcex.platform.imdg.api.ImdgProvider;
+import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
+import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
+
+public abstract class AbstractTable<T extends SpcexObjectBase> {
+    protected HazelcastService hazelcastService;
+    protected ImdgHazelcast<T> map;
+    private String prefix;
+    private Class<T> clazz;
+    private String nameOfMap;
+
+    protected AbstractTable(String prefix, Class<T> clazz, String nameOfMap) {
+        this.prefix = prefix;
+        this.clazz = clazz;
+        this.nameOfMap = nameOfMap;
+    }
+
+    public HazelcastService getHazelcastService() {
+        return hazelcastService;
+    }
+
+    public void setHazelcastService(HazelcastService hazelcastService) {
+        this.hazelcastService = hazelcastService;
+    }
+
+    public String getPrefix() {
+        return prefix;
+    }
+
+    public void setPrefix(String prefix) {
+        this.prefix = prefix;
+    }
+
+    public Class<? extends SpcexObjectBase> getClazz() {
+        return clazz;
+    }
+
+    public void setObject(Class<T> clazz) {
+        this.clazz = clazz;
+    }
+
+    public String getNameOfMap() {
+        return nameOfMap;
+    }
+
+    public void setNameOfMap(String nameOfMap) {
+        this.nameOfMap = nameOfMap;
+    }
+
+    public ImdgHazelcast<T> getImdg() {
+        return map;
+    }
+
+    public abstract T getEntity(Object[] entity);
+
+    public void injectEntity(T obj) {
+        map.insert(obj);
+    }
+
+    public ImdgHazelcast<T> getMap() {
+        if (map == null) {
+            bootMap();
+        }
+        return map;
+    }
+
+    private void bootMap() {
+        map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
+    }
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java

@@ -0,0 +1,37 @@
+package ru.spcex.clearing.dbf.importer.logic.data.tables;
+
+import ru.clearing.classes.statics.data.sdf.SDf01;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
+
+public class SDf01Table extends AbstractTable<SDf01> {
+
+    private static final String PREFIX = ETable.DF_01.name();
+    private static final Class<SDf01> CLAZZ = SDf01.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf01;
+
+    public SDf01Table() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
+    @Override
+    public SDf01 getEntity(Object[] entity) {
+        SDf01 result = new SDf01();
+
+        result.setId(this.getMap().nextIDSequenceFor());
+        result.setCurr_code((String) entity[0]);
+        result.setAccount((String) entity[1]);
+        result.setRemainder((String) entity[2]);
+        result.setDeal((String) entity[3]);
+        result.setAcc_code((String) entity[4]);
+        result.setDat((String) entity[5]);
+        result.setMarket((String) entity[6]);
+        result.setAcc_name((String) entity[7]);
+        result.setAcc_type((String) entity[8]);
+        result.setSumengage((String) entity[9]);
+        result.setSumunblock((String) entity[10]);
+        result.setFile_type((String) entity[11]);
+        return result;
+    }
+
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java

@@ -0,0 +1,43 @@
+package ru.spcex.clearing.dbf.importer.logic.data.tables;
+
+import ru.clearing.classes.statics.data.sdf.SDf02;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
+import ru.spcex.platform.utils.time.TimeUtil;
+
+public class SDf02Table extends AbstractTable<SDf02> {
+
+    private static final String PREFIX = ETable.DF_02.name();
+    private static final Class<SDf02> CLAZZ = SDf02.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf02;
+
+    public SDf02Table() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
+    @Override
+    public SDf02 getEntity(Object[] entity) {
+        SDf02 result = new SDf02();
+
+        result.setId(this.getMap().nextIDSequenceFor());
+        result.setCurr_code((String) entity[0]);
+        result.setAccount((String) entity[1]);
+        result.setRemainder((String) entity[2]);
+        result.setDeal((String) entity[3]);
+        result.setAcc_code((String) entity[4]);
+        result.setDat((String) entity[5]);
+        result.setMarket((String) entity[6]);
+        result.setAcc_name((String) entity[7]);
+        result.setAcc_type((String) entity[8]);
+        result.setSumengage((String) entity[9]);
+        result.setSumunblock((String) entity[10]);
+        result.setFile_type((String) entity[11]);
+        result.setResult((String) entity[12]);
+        result.setGenerationTime(TimeUtil.strToInstant((String) entity[13]));
+        result.setGenerationId((Long) entity[14]);
+        result.setInSDf01Id((Long) entity[15]);
+        return result;
+    }
+
+}
+

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java

@@ -0,0 +1,33 @@
+package ru.spcex.clearing.dbf.importer.logic.data.tables;
+
+import ru.clearing.classes.statics.data.sdf.SDf08;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
+import ru.spcex.platform.utils.time.TimeUtil;
+
+import java.math.BigDecimal;
+
+public class SDf08Table extends AbstractTable<SDf08> {
+
+    private static final String PREFIX = ETable.DF_08.name();
+    private static final Class<SDf08> CLAZZ = SDf08.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf08;
+
+
+    public SDf08Table() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
+    @Override
+    public SDf08 getEntity(Object[] entity) {
+        SDf08 result = new SDf08();
+        result.setId(this.getMap().nextIDSequenceFor());
+        result.setNumber((BigDecimal) entity[0]);
+        result.setDatetime(TimeUtil.strToInstant((String) entity[1]));
+        result.setGenerationTime(TimeUtil.strToInstant((String) entity[2]));
+        result.setGenerationId((Long) entity[3]);
+        return result;
+    }
+
+
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java

@@ -0,0 +1,34 @@
+package ru.spcex.clearing.dbf.importer.logic.data.tables;
+
+import ru.clearing.classes.statics.data.sdf.SDf12;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
+import ru.spcex.platform.utils.time.TimeUtil;
+
+import java.math.BigDecimal;
+
+public class SDf12Table extends AbstractTable<SDf12> {
+
+    private static final String PREFIX = ETable.DF_12.name();
+    private static final Class<SDf12> CLAZZ = SDf12.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf12;
+
+    public SDf12Table() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
+    @Override
+    public SDf12 getEntity(Object[] entity) {
+        SDf12 result = new SDf12();
+
+        result.setId(this.getMap().nextIDSequenceFor());
+        result.setAccount((String) entity[0]);
+        result.setDeal((String) entity[1]);
+        result.setStatus((BigDecimal) entity[2]);
+        result.setFileName((String) entity[3]);
+        result.setGenerationTime(TimeUtil.strToInstant((String) entity[4]));
+        result.setGenerationId((Long) entity[5]);
+
+        return result;
+    }
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java

@@ -0,0 +1,39 @@
+package ru.spcex.clearing.dbf.importer.logic.data.tables;
+
+import ru.clearing.classes.statics.data.sdf.SDf16;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
+import ru.spcex.platform.utils.time.TimeUtil;
+
+import java.math.BigDecimal;
+
+public class SDf16Table extends AbstractTable<SDf16> {
+
+    private static final String PREFIX = ETable.DF_16.name();
+    private static final Class<SDf16> CLAZZ = SDf16.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf16;
+
+    public SDf16Table() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
+    @Override
+    public SDf16 getEntity(Object[] entity) {
+        SDf16 result = new SDf16();
+
+        result.setId(this.getMap().nextIDSequenceFor());
+        result.setDate(TimeUtil.strToInstant((String) entity[0]));
+        result.setAccount((String) entity[1]);
+        result.setSum((BigDecimal) entity[2]);
+        result.setMarket((String) entity[3]);
+        result.setType((String) entity[4]);
+        result.setINN((BigDecimal) entity[5]);
+        result.setBIC((BigDecimal) entity[6]);
+        result.setSPEC((String) entity[7]);
+        result.setNumber((BigDecimal) entity[8]);
+        result.setFileName((String) entity[9]);
+        result.setGenerationTime(TimeUtil.strToInstant((String) entity[10]));
+        result.setGenerationId((Long) entity[11]);
+        return result;
+    }
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java

@@ -0,0 +1,33 @@
+package ru.spcex.clearing.dbf.importer.logic.data.tables;
+
+import ru.clearing.classes.statics.data.sdf.SDf18;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.imdg.IMDGDistributedNames;
+import ru.spcex.platform.utils.time.TimeUtil;
+
+import java.math.BigDecimal;
+
+public class SDf18Table extends AbstractTable<SDf18> {
+
+    private static final String PREFIX = ETable.DF_18.name();
+    private static final Class<SDf18> CLAZZ = SDf18.class;
+    private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf18;
+
+    public SDf18Table() {
+        super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
+    }
+
+    @Override
+    public SDf18 getEntity(Object[] entity) {
+        SDf18 result = new SDf18();
+        result.setId(this.getMap().nextIDSequenceFor());
+        result.setAccount((String) entity[0]);
+        result.setDeal((String) entity[1]);
+        result.setStatus((BigDecimal) entity[2]);
+        result.setResult((BigDecimal) entity[3]);
+        result.setGenerationTime(TimeUtil.strToInstant((String) entity[4]));
+        result.setGenerationId((Long) entity[5]);
+        result.setInSDf12Id((Long) entity[6]);
+        return result;
+    }
+}

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java

@@ -1,28 +1,21 @@
 package ru.spcex.clearing.dbf.importer.logic.stages;
 
-import com.linuxense.javadbf.DBFField;
 import com.linuxense.javadbf.DBFReader;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.dao.DataAccessException;
-import org.springframework.jdbc.core.BatchPreparedStatementSetter;
-import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.ColumnType;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
 import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable;
 import ru.spcex.clearing.dbf.importer.properties.AProperties;
+import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
-import java.sql.PreparedStatement;
+
-import java.sql.SQLException;
+import static ru.spcex.clearing.dbf.importer.utils.Const.MAPPING_ENUM_TABLE_OBJECT_TABLE;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
 
 /**
  * Заливка проверенных данных в базу
@@ -30,82 +23,32 @@ import java.util.Objects;
 @Component
 public class ImportToDB extends Stage {
     private final AProperties properties;
-    private final JdbcTemplate dbfJdbcTemplate;
+    private final HazelcastService hazelcastService;
 
     public ImportToDB(@Qualifier("dbfImporterProperties") AProperties properties,
-                      @Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) {
+                      HazelcastService hazelcastService) {
         this.properties = properties;
-        this.dbfJdbcTemplate = dbfJdbcTemplate;
+        this.hazelcastService = hazelcastService;
+
+
     }
 
     @Override
     public StageResult process(ResultContainer resultContainer) {
-        Objects.requireNonNull(resultContainer.getDbfTable());
+        ETable currTable = resultContainer.getDbfTable();
-        Objects.requireNonNull(resultContainer.getDbfSource());
-
-        Table currTable = resultContainer.getDbfTable();
         byte[] source = resultContainer.getDbfSource();
         Charset sourceCharset = Charset.forName(properties.getDbfEncoding());
 
         try (InputStream is = new ByteArrayInputStream(source);
              DBFReader dbfReader = new DBFReader(is, sourceCharset)) {
-            int columnCount = dbfReader.getFieldCount();
+            AbstractTable table = MAPPING_ENUM_TABLE_OBJECT_TABLE.get(currTable);
-
+            table.setHazelcastService(hazelcastService);
-            List<String> columnNames = new ArrayList<>(columnCount);
+            for (int i = 0; i < dbfReader.getRecordCount(); i++) {
-            List<Integer> columnTypes = new ArrayList<>(columnCount);
+                Object[] entity = dbfReader.nextRecord();
-            for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) {
+                table.injectEntity(table.getEntity(entity));
-                DBFField currField = dbfReader.getField(columnIdx);
-                String fieldName = currField.getName();
-                ColumnType columnType = ColumnType.getForDBFType(currField.getType());
-                columnNames.add(fieldName);
-                columnTypes.add(columnType.getSqlType());
             }
-
+        } catch (IOException exception) {
-            int recordCount = dbfReader.getRecordCount();
+            log.warn(exception.getMessage());
-            List<Object[]> values = new ArrayList<>(recordCount);
-            for (int rowIdx = 0; rowIdx < recordCount; rowIdx++) {
-                Object[] dbfRow = dbfReader.nextRecord();
-                values.add(dbfRow);
-            }
-
-            int maxBatchSize = properties.getInsertBatchSize();
-            int batchCount = recordCount / maxBatchSize;
-            int batchRem = recordCount % maxBatchSize;
-            for (int batchIdx = 0; batchIdx < batchCount + 1; batchIdx++) {
-                if (batchIdx == batchCount && batchRem == 0) continue;
-                final Integer currentBatchIdx = batchIdx;
-
-                int[] rowsInserted = dbfJdbcTemplate.batchUpdate(
-                        "insert into " + currTable.name() +
-                                " (" + String.join(",", columnNames) + ")" +
-                                " values (" + String.join(",", Collections.nCopies(columnCount, "?")) + ")",
-                        new BatchPreparedStatementSetter() {
-                            @Override
-                            public void setValues(PreparedStatement ps, int i) throws SQLException {
-                                Object[] currValuesRow = values.get(currentBatchIdx * maxBatchSize + i);
-                                int parameterIndex = 1;
-                                for (Object object : currValuesRow) {
-                                    ps.setObject(parameterIndex, object, columnTypes.get(parameterIndex - 1));
-                                    parameterIndex++;
-                                }
-                            }
-
-                            @Override
-                            public int getBatchSize() {
-                                return currentBatchIdx == batchCount ? batchRem : maxBatchSize;
-                            }
-                        });
-                log.info("uuid {}. Imported records: {}", resultContainer.getUuid(), rowsInserted.length);
-            }
-
-        } catch (IOException e) {
-            log.error(String.format("uuid %s. Can't read source file %s.",
-                    resultContainer.getUuid(),
-                    resultContainer.getDbfFile().getName()), e);
-            return StageResult.ERROR;
-        } catch (DataAccessException e) {
-            log.error(String.format("uuid %s. Can't insert records", resultContainer.getUuid()), e);
-            return StageResult.ERROR;
         }
 
 

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java

@@ -1,194 +1,39 @@
 package ru.spcex.clearing.dbf.importer.logic.stages;
 
-import com.linuxense.javadbf.DBFDataType;
-import com.linuxense.javadbf.DBFField;
-import com.linuxense.javadbf.DBFReader;
 import org.springframework.beans.factory.InitializingBean;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.jdbc.support.JdbcUtils;
-import org.springframework.jdbc.support.MetaDataAccessException;
 import org.springframework.stereotype.Component;
 import ru.spcex.clearing.dbf.importer.exceptions.ConfigException;
-import ru.spcex.clearing.dbf.importer.logic.data.ColumnStructureDBF;
-import ru.spcex.clearing.dbf.importer.logic.data.DBStructureDBF;
 import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
-import ru.spcex.clearing.dbf.importer.logic.data.TableStructureDBF;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.ColumnType;
 import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
 import ru.spcex.clearing.dbf.importer.properties.AProperties;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
 import java.nio.charset.Charset;
-import java.sql.ResultSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
 
 /**
  * Фильтрация источника на предмет соответствия полей
  */
 @Component
 public class ValidateFields extends Stage implements InitializingBean {
-    private final JdbcTemplate dbfJdbcTemplate;
     private final AProperties properties;
-
-    private DBStructureDBF dbStructureDBF;
     private Charset dbfCharset;
 
-    public ValidateFields(@Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate, AProperties properties) {
+    public ValidateFields(AProperties properties) {
-        this.dbfJdbcTemplate = dbfJdbcTemplate;
         this.properties = properties;
     }
 
     @Override
     public StageResult process(ResultContainer resultContainer) {
-        Objects.requireNonNull(resultContainer.getDbfTable());
-        Objects.requireNonNull(resultContainer.getDbfSource());
-        UUID taskUuid = resultContainer.getUuid();
-        Table currTable = resultContainer.getDbfTable();
-        byte[] source = resultContainer.getDbfSource();
-
-        TableStructureDBF currTableStructure = dbStructureDBF.getTables().get(currTable);
-        if (currTableStructure == null) {
-            log.error("uuid {}. Unknown table {}.", taskUuid, currTable);
-            return StageResult.ERROR;
-        }
-
-        log.info("uuid {}. Check source {}", taskUuid, currTable);
-
-        try (InputStream is = new ByteArrayInputStream(source);
-             DBFReader dbfReader = new DBFReader(is, dbfCharset)) {
-
-            int columnCount = dbfReader.getFieldCount();
-            if (columnCount != currTableStructure.getColumns().size()) {
-                log.error("uuid {}. Source for table {} doesn't match DB columns count. Source column count: {}. DB column count: {}.",
-                        taskUuid,
-                        currTable,
-                        columnCount,
-                        currTableStructure.getColumns().size());
-                return StageResult.ERROR;
-            }
-
-            boolean valid = true;
-            for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) {
-                DBFField currField = dbfReader.getField(columnIdx);
-
-                String currFieldName = currField.getName();
-                ColumnStructureDBF currColumnStructure = currTableStructure.getIgnoreCase(currFieldName);
-                if (currColumnStructure == null) {
-                    log.error("uuid {}. Source for table {} contain unknown field {}. ",
-                            taskUuid,
-                            currTable,
-                            currFieldName);
-                    valid = false;
-                    continue;
-                }
-
-                DBFDataType dbfDataType = currField.getType();
-                if (!currColumnStructure.getType().getDbfType().equals(dbfDataType)) {
-                    log.error("uuid {}. Field {} from source {} has wrong data type (actual {}, expected {}).",
-                            taskUuid,
-                            currFieldName,
-                            currTable,
-                            dbfDataType != null ? dbfDataType.name() : "null",
-                            currColumnStructure.getType());
-                    valid = false;
-                    continue;
-                }
-
-                int fieldLength = currField.getLength();
-                if (fieldLength != currColumnStructure.getLength()) {
-
-                    if (!dbfDataType.equals(DBFDataType.DATE)) {
-                        log.error("uuid {}. Field {} from source {} has wrong length (actual {}, expected {}).",
-                                taskUuid,
-                                currFieldName,
-                                currTable,
-                                fieldLength,
-                                currColumnStructure.getLength());
-                        valid = false;
-                        continue;
-                    } else {
-                        if (fieldLength > currColumnStructure.getLength()) {
-                            log.error("uuid {}. Field {} with DATE type has wrong length (length from file > length db field)",
-                                    taskUuid,
-                                    currFieldName);
-                            valid = false;
-                            continue;
-                        }
-                    }
-                }
-
-                int decimalCount = currField.getDecimalCount();
-                if (decimalCount != currColumnStructure.getDecimalDigits()) {
-                    log.error("uuid {}. Field {} from source {} has wrong decimal count (actual {}, expected {}).",
-                            taskUuid,
-                            currFieldName,
-                            currTable,
-                            decimalCount,
-                            currColumnStructure.getDecimalDigits());
-                    valid = false;
-                }
-
-                if (!valid) {
-                    return StageResult.ERROR;
-                }
-            }
-        } catch (IOException e) {
-            log.error(String.format("uuid %s. Can't read source for table %s.", taskUuid, currTable), e);
-            return StageResult.ERROR;
-        }
-
         return StageResult.OK;
     }
 
-
     /**
-     * 1. Собирает из БД названия столбцов для дальнейшей валидации
+     * 1. Проверяет кодировку из настройки dbf.encoding-source
-     * 2. Проверяет кодировку из настройки dbf.encoding-source
      */
     @Override
     public void afterPropertiesSet() throws Exception {
-        initDBStructure();
         initDBFCharset();
     }
 
-    private void initDBStructure() throws MetaDataAccessException {
-        dbStructureDBF = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()),
-                dbMeta -> {
-                    DBStructureDBF dbStructureDBF = new DBStructureDBF();
-                    List<String> tableNamesFromResultSet = new LinkedList<>();
-                    ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"});
-                    while (tableNamesRS.next()) {
-                        tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME"));
-                    }
-                    for (String tableNameFromResultSet : tableNamesFromResultSet) {
-                        Table currTable = Table.tableForName(tableNameFromResultSet);
-                        if (currTable == null) continue;
-                        TableStructureDBF tableStructureDBF = new TableStructureDBF();
-                        ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null);
-                        while (columnNamesRS.next()) {
-                            ColumnStructureDBF columnStructureDBF = new ColumnStructureDBF();
-                            columnStructureDBF.setName(columnNamesRS.getString("COLUMN_NAME"));
-                            columnStructureDBF.setType(ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE")));
-                            columnStructureDBF.setLength(columnNamesRS.getInt("COLUMN_SIZE"));
-                            columnStructureDBF.setDecimalDigits(columnNamesRS.getInt("DECIMAL_DIGITS"));
-                            columnStructureDBF.setComment(columnNamesRS.getString("REMARKS"));
-                            tableStructureDBF.getColumns().put(columnStructureDBF.getName(), columnStructureDBF);
-                        }
-                        assert !dbStructureDBF.getTables().containsKey(currTable);
-
-                        dbStructureDBF.getTables().put(currTable, tableStructureDBF);
-                    }
-                    return dbStructureDBF;
-                });
-    }
-
     private void initDBFCharset() {
         try {
             dbfCharset = Charset.forName(properties.getDbfEncoding());
@@ -196,5 +41,4 @@ public class ValidateFields extends Stage implements InitializingBean {
             throw new ConfigException("Unknown encoding from properties: " + properties.getDbfEncoding(), e);
         }
     }
-
 }

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java

@@ -6,7 +6,7 @@ import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
 import ru.spcex.clearing.dbf.importer.logic.stages.Processor;
 
 import java.io.File;
@@ -30,9 +30,9 @@ public class DBFImporterService {
 
     @Scheduled(cron = "${dbf.check-src-dir-cron}")
     public void run() {
-        Map<Table, List<File>> newFiles = fileChecker.checkNewFiles();
+        Map<ETable, List<File>> newFiles = fileChecker.checkNewFiles();
-        for (Map.Entry<Table, List<File>> newFilesEntry : newFiles.entrySet()) {
+        for (Map.Entry<ETable, List<File>> newFilesEntry : newFiles.entrySet()) {
-            Table currTable = newFilesEntry.getKey();
+            ETable currTable = newFilesEntry.getKey();
             List<File> fileList = newFilesEntry.getValue();
             for (File dbfFile : fileList) {
                 executorService.execute(() -> processor.process(ResultContainer.createNewTask(currTable, dbfFile)));

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java

@@ -1,7 +1,7 @@
 package ru.spcex.clearing.dbf.importer.services;
 
 import org.springframework.stereotype.Service;
-import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
 import ru.spcex.clearing.dbf.importer.properties.AProperties;
 
 import java.io.File;
@@ -15,8 +15,8 @@ public class FileChecker {
         this.properties = properties;
     }
 
-    public Map<Table, List<File>> checkNewFiles() {
+    public Map<ETable, List<File>> checkNewFiles() {
-        Map<Table, List<File>> newFiles = new EnumMap<>(Table.class);
+        Map<ETable, List<File>> newFiles = new EnumMap<>(ETable.class);
 
         String srcDir = properties.getSrcDir();
         List<File> dbfFiles = lsDBF(srcDir);
@@ -25,7 +25,7 @@ public class FileChecker {
         for (File dbfFile : dbfFiles) {
             if (dbfFile.isDirectory()) continue;
 
-            Table currTable = Table.getTableForFilename(dbfFile.getName());
+            ETable currTable = ETable.getTableForFilename(dbfFile.getName());
             if (currTable == null) continue;
 
             List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());

clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/utils/Const.java

@@ -0,0 +1,23 @@
+package ru.spcex.clearing.dbf.importer.utils;
+
+import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
+import ru.spcex.clearing.dbf.importer.logic.data.tables.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Const {
+
+    public static final Map<ETable, AbstractTable> MAPPING_ENUM_TABLE_OBJECT_TABLE = new HashMap<>();
+
+    static {
+        MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_01, new SDf01Table());
+        MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_02, new SDf02Table());
+        MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_08, new SDf08Table());
+        MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_12, new SDf12Table());
+        MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_16, new SDf16Table());
+        MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_18, new SDf18Table());
+    }
+
+
+}

clearing-parent/dbf-importer/src/main/resources/application.properties

@@ -1,17 +1,16 @@
 server.port=8080
 server.servlet.context-path=/importer
 spring.main.web-application-type=servlet
-
 db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres
 db.driver=org.postgresql.Driver
 db.login=clearing
 db.password=Aa111111
-
 dbf.check-src-dir-cron=* * * * 1 ?
 dbf.encoding-source=cp866
 dbf.insert-batch-size=100
-
 dbf.delete-src-files=false
 dbf.src-dir=D:\\dbf\\
-
+dbf.threads-count=10
-dbf.threads-count=10
+import-dbf-service.hazelcast.cluster-members=127.0.0.1
+import-dbf-service.hazelcast.login=dev
+import-dbf-service.hazelcast.password=dev-pass

platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java

@@ -13,6 +13,7 @@ import ru.spcex.platform.imdg.api.Imdg;
 import ru.spcex.platform.utils.log.ExceptionUtils;
 
 import java.util.*;
+
 public class ImdgHazelcast<T extends SpcexObjectBase> implements Imdg<T> {
     private final Logger log = LoggerFactory.getLogger(getClass());
     private IdGenerator idGenerator;
@@ -109,7 +110,7 @@ public class ImdgHazelcast<T extends SpcexObjectBase> implements Imdg<T> {
         Set<Long> ids = map.keySet(or);
         Iterator<Long> idIterator = ids.iterator();
         Collection<T> searchResult = new ArrayList<>();
-        while(idIterator.hasNext()) {
+        while (idIterator.hasNext()) {
             T element = map.get(idIterator.next());
             if (element != null) {
                 searchResult.add(element);
@@ -118,6 +119,11 @@ public class ImdgHazelcast<T extends SpcexObjectBase> implements Imdg<T> {
         return searchResult;
     }
 
+    @Override
+    public Long nextIDSequenceFor() {
+        return idGenerator.newId();
+    }
+
     public IMap<Long, T> getMap() {
         return map;
     }

platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgTransactionalHazelcast.java

@@ -105,7 +105,7 @@ public class ImdgTransactionalHazelcast<T extends SpcexObjectBase> implements Im
         Set<Long> ids = map.keySet(or);
         Iterator<Long> idIterator = ids.iterator();
         Collection<T> searchResult = new ArrayList<>();
-        while(idIterator.hasNext()) {
+        while (idIterator.hasNext()) {
             T element = map.get(idIterator.next());
             if (element != null) {
                 searchResult.add(element);
@@ -145,4 +145,9 @@ public class ImdgTransactionalHazelcast<T extends SpcexObjectBase> implements Im
     public void setMapName(String mapName) {
         this.mapName = mapName;
     }
+
+    @Override
+    public Long nextIDSequenceFor() {
+        return idGenerator.newId();
+    }
 }

platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/time/TimeUtil.java

@@ -4,8 +4,10 @@ import java.time.Instant;
 import java.time.LocalDate;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Objects;
 
 public class TimeUtil {
+    public static final DateTimeFormatter PROPERTY_DATE_FORMATTER = DateTimeFormatter.ofPattern("dd.MM.yyyy");
     public static ZoneId zone = ZoneId.systemDefault();
 
     /**
@@ -27,4 +29,17 @@ public class TimeUtil {
     public static Instant localDateToInstant(LocalDate date) {
         return date.atStartOfDay(zone).toInstant();
     }
+
+    public static LocalDate strToLocalDate(String date) {
+        try {
+            LocalDate parsed = LocalDate.parse(date, PROPERTY_DATE_FORMATTER);
+            return parsed;
+        } catch (Throwable e) {
+            return null;
+        }
+    }
+
+    public static Instant strToInstant(String date) {
+        return localDateToInstant(Objects.requireNonNull(strToLocalDate(date)));
+    }
 }