treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

importer almost done

Browse source
This commit is contained in:
aalehin 2022-09-21 16:58:49 +03:00
parent cc8a36018d
commit 7759391a15
23 changed files with 496 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
clearing-parent/dbf-importer/pom.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@ -50,9 +50,30 @@
<groupId>com.github.albfernandez</groupId>
<artifactId>javadbf</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-imdg-api</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-imdg-api-hazelcast-impl</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>classes</artifactId>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>application.properties</exclude>
</excludes>
<filtering>false</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
@ -69,6 +90,7 @@
</configuration>
</plugin>
</plugins>
</build>
</project>

1
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java
View file

@ -27,6 +27,7 @@ public class DBFImporterConfig {
private final AProperties properties;
private final ApplicationContext context;
public DBFImporterConfig(@Qualifier("dbfImporterProperties") AProperties properties, ApplicationContext context) {
this.properties = properties;
this.context = context;

46
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/ImporterImdgConfig.java Normal file
View file

@ -0,0 +1,46 @@
package ru.spcex.clearing.dbf.importer.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import ru.spcex.clearing.dbf.importer.config.settings.ImportDBFServiceSettings;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
@Configuration
public class ImporterImdgConfig {
private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) {
ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
if (maxPoolSz > 2) {
pool.setKeepAliveSeconds(60);
pool.setAllowCoreThreadTimeOut(true);
}
pool.setCorePoolSize(maxPoolSz);
pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion);
return pool;
}
@Bean(name = "taskExecutorHazelcastClientInitializer")
public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() {
return createThreadPoolTaskExecutor(1, true);
}
@Bean(name = "taskExecutorIdGeneratorAwaiter")
public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() {
return createThreadPoolTaskExecutor(1, false);
}
@Autowired
@Bean
public HazelcastService imdgProvider(
@Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
@Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
ImportDBFServiceSettings settings
) {
return new HazelcastService(taskExecutorHazelcastClientInitializer,
taskExecutorIdGeneratorAwaiter,
settings.getHazelcast());
}
}

21
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/settings/ImportDBFServiceSettings.java Normal file
View file

@ -0,0 +1,21 @@
package ru.spcex.clearing.dbf.importer.config.settings;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams;
@Component
@PropertySource("file:${spring.config.location}/application.properties")
@ConfigurationProperties("import-dbf-service")
public class ImportDBFServiceSettings {
private HazelcastClientParams hazelcast;
public HazelcastClientParams getHazelcast() {
return hazelcast;
}
public void setHazelcast(HazelcastClientParams hazelcast) {
this.hazelcast = hazelcast;
}
}

8
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java
View file

@ -1,18 +1,18 @@
package ru.spcex.clearing.dbf.importer.logic.data;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import java.util.EnumMap;
import java.util.Map;
public class DBStructureDBF {
private Map<Table, TableStructureDBF> tables = new EnumMap<>(Table.class);
private Map<ETable, TableStructureDBF> tables = new EnumMap<>(ETable.class);
public Map<Table, TableStructureDBF> getTables() {
public Map<ETable, TableStructureDBF> getTables() {
return tables;
}
public void setTables(Map<Table, TableStructureDBF> tables) {
public void setTables(Map<ETable, TableStructureDBF> tables) {
this.tables = tables;
}
}

13
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/ResultContainer.java
View file

@ -1,6 +1,6 @@
package ru.spcex.clearing.dbf.importer.logic.data;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import java.io.File;
import java.util.UUID;
@ -10,14 +10,15 @@ import java.util.UUID;
*/
public class ResultContainer {
private UUID uuid;
private Table dbfTable;
private ETable dbfTable;
private File dbfFile;
private byte[] dbfSource;
protected ResultContainer() {}
protected ResultContainer() {
}
public static ResultContainer createNewTask(Table dbfTable, File dbfFile) {
public static ResultContainer createNewTask(ETable dbfTable, File dbfFile) {
ResultContainer container = new ResultContainer();
container.dbfTable = dbfTable;
container.dbfFile = dbfFile;
@ -25,11 +26,11 @@ public class ResultContainer {
return container;
}
public Table getDbfTable() {
public ETable getDbfTable() {
return dbfTable;
}
public void setDbfTable(Table dbfTable) {
public void setDbfTable(ETable dbfTable) {
this.dbfTable = dbfTable;
}

30
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/Table.java → clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/enums/ETable.java
View file

@ -1,35 +1,41 @@
package ru.spcex.clearing.dbf.importer.logic.data.enums;
public enum Table {
public enum ETable {
DF_01("DF-01"),
DF_02("DF-02"),
DF_03("DF-03"),
DF_04("DF-04"),
DF_05("DF-05"),
DF_06("DF-06");
DF_08("DF-08"),
DF_09("DF-09"),
DF_10("DF-10"),
DF_11("DF-11"),
DF_12("DF-12"),
DF_16("DF-16"),
DF_17("DF-17"),
DF_18("DF-18");
private final String prefix;
Table(String prefix) {
ETable(String prefix) {
this.prefix = prefix;
}
public boolean fileForThisTable(String filename) {
return filename != null && filename.startsWith(prefix);
}
public static Table getTableForFilename(String filename) {
for (Table table : Table.values()) {
public static ETable getTableForFilename(String filename) {
for (ETable table : ETable.values()) {
if (table.fileForThisTable(filename)) return table;
}
return null;
}
public static Table tableForName(String name) {
for (Table table : values()) {
public static ETable tableForName(String name) {
for (ETable table : values()) {
if (name.equalsIgnoreCase(table.name()))
return table;
}
return null;
}
public boolean fileForThisTable(String filename) {
return filename != null && filename.startsWith(prefix);
}
}

73
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/AbstractTable.java Normal file
View file

@ -0,0 +1,73 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.imdg.api.ImdgProvider;
import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
public abstract class AbstractTable<T extends SpcexObjectBase> {
protected HazelcastService hazelcastService;
protected ImdgHazelcast<T> map;
private String prefix;
private Class<T> clazz;
private String nameOfMap;
protected AbstractTable(String prefix, Class<T> clazz, String nameOfMap) {
this.prefix = prefix;
this.clazz = clazz;
this.nameOfMap = nameOfMap;
}
public HazelcastService getHazelcastService() {
return hazelcastService;
}
public void setHazelcastService(HazelcastService hazelcastService) {
this.hazelcastService = hazelcastService;
}
public String getPrefix() {
return prefix;
}
public void setPrefix(String prefix) {
this.prefix = prefix;
}
public Class<? extends SpcexObjectBase> getClazz() {
return clazz;
}
public void setObject(Class<T> clazz) {
this.clazz = clazz;
}
public String getNameOfMap() {
return nameOfMap;
}
public void setNameOfMap(String nameOfMap) {
this.nameOfMap = nameOfMap;
}
public ImdgHazelcast<T> getImdg() {
return map;
}
public abstract T getEntity(Object[] entity);
public void injectEntity(T obj) {
map.insert(obj);
}
public ImdgHazelcast<T> getMap() {
if (map == null) {
bootMap();
}
return map;
}
private void bootMap() {
map = (ImdgHazelcast<T>) hazelcastService.getImdg(nameOfMap, clazz);
}
}

37
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf01Table.java Normal file
View file

@ -0,0 +1,37 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import ru.clearing.classes.statics.data.sdf.SDf01;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
public class SDf01Table extends AbstractTable<SDf01> {
private static final String PREFIX = ETable.DF_01.name();
private static final Class<SDf01> CLAZZ = SDf01.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf01;
public SDf01Table() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@Override
public SDf01 getEntity(Object[] entity) {
SDf01 result = new SDf01();
result.setId(this.getMap().nextIDSequenceFor());
result.setCurr_code((String) entity[0]);
result.setAccount((String) entity[1]);
result.setRemainder((String) entity[2]);
result.setDeal((String) entity[3]);
result.setAcc_code((String) entity[4]);
result.setDat((String) entity[5]);
result.setMarket((String) entity[6]);
result.setAcc_name((String) entity[7]);
result.setAcc_type((String) entity[8]);
result.setSumengage((String) entity[9]);
result.setSumunblock((String) entity[10]);
result.setFile_type((String) entity[11]);
return result;
}
}

43
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf02Table.java Normal file
View file

@ -0,0 +1,43 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import ru.clearing.classes.statics.data.sdf.SDf02;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.platform.utils.time.TimeUtil;
public class SDf02Table extends AbstractTable<SDf02> {
private static final String PREFIX = ETable.DF_02.name();
private static final Class<SDf02> CLAZZ = SDf02.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf02;
public SDf02Table() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@Override
public SDf02 getEntity(Object[] entity) {
SDf02 result = new SDf02();
result.setId(this.getMap().nextIDSequenceFor());
result.setCurr_code((String) entity[0]);
result.setAccount((String) entity[1]);
result.setRemainder((String) entity[2]);
result.setDeal((String) entity[3]);
result.setAcc_code((String) entity[4]);
result.setDat((String) entity[5]);
result.setMarket((String) entity[6]);
result.setAcc_name((String) entity[7]);
result.setAcc_type((String) entity[8]);
result.setSumengage((String) entity[9]);
result.setSumunblock((String) entity[10]);
result.setFile_type((String) entity[11]);
result.setResult((String) entity[12]);
result.setGenerationTime(TimeUtil.strToInstant((String) entity[13]));
result.setGenerationId((Long) entity[14]);
result.setInSDf01Id((Long) entity[15]);
return result;
}
}

33
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf08Table.java Normal file
View file

@ -0,0 +1,33 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import ru.clearing.classes.statics.data.sdf.SDf08;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.platform.utils.time.TimeUtil;
import java.math.BigDecimal;
public class SDf08Table extends AbstractTable<SDf08> {
private static final String PREFIX = ETable.DF_08.name();
private static final Class<SDf08> CLAZZ = SDf08.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf08;
public SDf08Table() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@Override
public SDf08 getEntity(Object[] entity) {
SDf08 result = new SDf08();
result.setId(this.getMap().nextIDSequenceFor());
result.setNumber((BigDecimal) entity[0]);
result.setDatetime(TimeUtil.strToInstant((String) entity[1]));
result.setGenerationTime(TimeUtil.strToInstant((String) entity[2]));
result.setGenerationId((Long) entity[3]);
return result;
}
}

34
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf12Table.java Normal file
View file

@ -0,0 +1,34 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import ru.clearing.classes.statics.data.sdf.SDf12;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.platform.utils.time.TimeUtil;
import java.math.BigDecimal;
public class SDf12Table extends AbstractTable<SDf12> {
private static final String PREFIX = ETable.DF_12.name();
private static final Class<SDf12> CLAZZ = SDf12.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf12;
public SDf12Table() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@Override
public SDf12 getEntity(Object[] entity) {
SDf12 result = new SDf12();
result.setId(this.getMap().nextIDSequenceFor());
result.setAccount((String) entity[0]);
result.setDeal((String) entity[1]);
result.setStatus((BigDecimal) entity[2]);
result.setFileName((String) entity[3]);
result.setGenerationTime(TimeUtil.strToInstant((String) entity[4]));
result.setGenerationId((Long) entity[5]);
return result;
}
}

39
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf16Table.java Normal file
View file

@ -0,0 +1,39 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import ru.clearing.classes.statics.data.sdf.SDf16;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.platform.utils.time.TimeUtil;
import java.math.BigDecimal;
public class SDf16Table extends AbstractTable<SDf16> {
private static final String PREFIX = ETable.DF_16.name();
private static final Class<SDf16> CLAZZ = SDf16.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf16;
public SDf16Table() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@Override
public SDf16 getEntity(Object[] entity) {
SDf16 result = new SDf16();
result.setId(this.getMap().nextIDSequenceFor());
result.setDate(TimeUtil.strToInstant((String) entity[0]));
result.setAccount((String) entity[1]);
result.setSum((BigDecimal) entity[2]);
result.setMarket((String) entity[3]);
result.setType((String) entity[4]);
result.setINN((BigDecimal) entity[5]);
result.setBIC((BigDecimal) entity[6]);
result.setSPEC((String) entity[7]);
result.setNumber((BigDecimal) entity[8]);
result.setFileName((String) entity[9]);
result.setGenerationTime(TimeUtil.strToInstant((String) entity[10]));
result.setGenerationId((Long) entity[11]);
return result;
}
}

33
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/tables/SDf18Table.java Normal file
View file

@ -0,0 +1,33 @@
package ru.spcex.clearing.dbf.importer.logic.data.tables;
import ru.clearing.classes.statics.data.sdf.SDf18;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.platform.utils.time.TimeUtil;
import java.math.BigDecimal;
public class SDf18Table extends AbstractTable<SDf18> {
private static final String PREFIX = ETable.DF_18.name();
private static final Class<SDf18> CLAZZ = SDf18.class;
private static final String NAME_OF_HZ_MAP = IMDGDistributedNames.Map_SDf18;
public SDf18Table() {
super(PREFIX, CLAZZ, NAME_OF_HZ_MAP);
}
@Override
public SDf18 getEntity(Object[] entity) {
SDf18 result = new SDf18();
result.setId(this.getMap().nextIDSequenceFor());
result.setAccount((String) entity[0]);
result.setDeal((String) entity[1]);
result.setStatus((BigDecimal) entity[2]);
result.setResult((BigDecimal) entity[3]);
result.setGenerationTime(TimeUtil.strToInstant((String) entity[4]));
result.setGenerationId((Long) entity[5]);
result.setInSDf12Id((Long) entity[6]);
return result;
}
}

93
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java
View file

@ -1,28 +1,21 @@
package ru.spcex.clearing.dbf.importer.logic.stages;
import com.linuxense.javadbf.DBFField;
import com.linuxense.javadbf.DBFReader;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ColumnType;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.data.tables.AbstractTable;
import ru.spcex.clearing.dbf.importer.properties.AProperties;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import static ru.spcex.clearing.dbf.importer.utils.Const.MAPPING_ENUM_TABLE_OBJECT_TABLE;
/**
* Заливка проверенных данных в базу
@ -30,82 +23,32 @@ import java.util.Objects;
@Component
public class ImportToDB extends Stage {
private final AProperties properties;
private final JdbcTemplate dbfJdbcTemplate;
private final HazelcastService hazelcastService;
public ImportToDB(@Qualifier("dbfImporterProperties") AProperties properties,
@Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) {
HazelcastService hazelcastService) {
this.properties = properties;
this.dbfJdbcTemplate = dbfJdbcTemplate;
this.hazelcastService = hazelcastService;
}
@Override
public StageResult process(ResultContainer resultContainer) {
Objects.requireNonNull(resultContainer.getDbfTable());
Objects.requireNonNull(resultContainer.getDbfSource());
Table currTable = resultContainer.getDbfTable();
ETable currTable = resultContainer.getDbfTable();
byte[] source = resultContainer.getDbfSource();
Charset sourceCharset = Charset.forName(properties.getDbfEncoding());
try (InputStream is = new ByteArrayInputStream(source);
DBFReader dbfReader = new DBFReader(is, sourceCharset)) {
int columnCount = dbfReader.getFieldCount();
List<String> columnNames = new ArrayList<>(columnCount);
List<Integer> columnTypes = new ArrayList<>(columnCount);
for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) {
DBFField currField = dbfReader.getField(columnIdx);
String fieldName = currField.getName();
ColumnType columnType = ColumnType.getForDBFType(currField.getType());
columnNames.add(fieldName);
columnTypes.add(columnType.getSqlType());
AbstractTable table = MAPPING_ENUM_TABLE_OBJECT_TABLE.get(currTable);
table.setHazelcastService(hazelcastService);
for (int i = 0; i < dbfReader.getRecordCount(); i++) {
Object[] entity = dbfReader.nextRecord();
table.injectEntity(table.getEntity(entity));
}
int recordCount = dbfReader.getRecordCount();
List<Object[]> values = new ArrayList<>(recordCount);
for (int rowIdx = 0; rowIdx < recordCount; rowIdx++) {
Object[] dbfRow = dbfReader.nextRecord();
values.add(dbfRow);
}
int maxBatchSize = properties.getInsertBatchSize();
int batchCount = recordCount / maxBatchSize;
int batchRem = recordCount % maxBatchSize;
for (int batchIdx = 0; batchIdx < batchCount + 1; batchIdx++) {
if (batchIdx == batchCount && batchRem == 0) continue;
final Integer currentBatchIdx = batchIdx;
int[] rowsInserted = dbfJdbcTemplate.batchUpdate(
"insert into " + currTable.name() +
" (" + String.join(",", columnNames) + ")" +
" values (" + String.join(",", Collections.nCopies(columnCount, "?")) + ")",
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
Object[] currValuesRow = values.get(currentBatchIdx * maxBatchSize + i);
int parameterIndex = 1;
for (Object object : currValuesRow) {
ps.setObject(parameterIndex, object, columnTypes.get(parameterIndex - 1));
parameterIndex++;
}
}
@Override
public int getBatchSize() {
return currentBatchIdx == batchCount ? batchRem : maxBatchSize;
}
});
log.info("uuid {}. Imported records: {}", resultContainer.getUuid(), rowsInserted.length);
}
} catch (IOException e) {
log.error(String.format("uuid %s. Can't read source file %s.",
resultContainer.getUuid(),
resultContainer.getDbfFile().getName()), e);
return StageResult.ERROR;
} catch (DataAccessException e) {
log.error(String.format("uuid %s. Can't insert records", resultContainer.getUuid()), e);
return StageResult.ERROR;
} catch (IOException exception) {
log.warn(exception.getMessage());
}

160
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java
View file

@ -1,194 +1,39 @@
package ru.spcex.clearing.dbf.importer.logic.stages;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import com.linuxense.javadbf.DBFReader;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.JdbcUtils;
import org.springframework.jdbc.support.MetaDataAccessException;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.importer.exceptions.ConfigException;
import ru.spcex.clearing.dbf.importer.logic.data.ColumnStructureDBF;
import ru.spcex.clearing.dbf.importer.logic.data.DBStructureDBF;
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.TableStructureDBF;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ColumnType;
import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.properties.AProperties;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.sql.ResultSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
/**
* Фильтрация источника на предмет соответствия полей
*/
@Component
public class ValidateFields extends Stage implements InitializingBean {
private final JdbcTemplate dbfJdbcTemplate;
private final AProperties properties;
private DBStructureDBF dbStructureDBF;
private Charset dbfCharset;
public ValidateFields(@Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate, AProperties properties) {
this.dbfJdbcTemplate = dbfJdbcTemplate;
public ValidateFields(AProperties properties) {
this.properties = properties;
}
@Override
public StageResult process(ResultContainer resultContainer) {
Objects.requireNonNull(resultContainer.getDbfTable());
Objects.requireNonNull(resultContainer.getDbfSource());
UUID taskUuid = resultContainer.getUuid();
Table currTable = resultContainer.getDbfTable();
byte[] source = resultContainer.getDbfSource();
TableStructureDBF currTableStructure = dbStructureDBF.getTables().get(currTable);
if (currTableStructure == null) {
log.error("uuid {}. Unknown table {}.", taskUuid, currTable);
return StageResult.ERROR;
}
log.info("uuid {}. Check source {}", taskUuid, currTable);
try (InputStream is = new ByteArrayInputStream(source);
DBFReader dbfReader = new DBFReader(is, dbfCharset)) {
int columnCount = dbfReader.getFieldCount();
if (columnCount != currTableStructure.getColumns().size()) {
log.error("uuid {}. Source for table {} doesn't match DB columns count. Source column count: {}. DB column count: {}.",
taskUuid,
currTable,
columnCount,
currTableStructure.getColumns().size());
return StageResult.ERROR;
}
boolean valid = true;
for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) {
DBFField currField = dbfReader.getField(columnIdx);
String currFieldName = currField.getName();
ColumnStructureDBF currColumnStructure = currTableStructure.getIgnoreCase(currFieldName);
if (currColumnStructure == null) {
log.error("uuid {}. Source for table {} contain unknown field {}. ",
taskUuid,
currTable,
currFieldName);
valid = false;
continue;
}
DBFDataType dbfDataType = currField.getType();
if (!currColumnStructure.getType().getDbfType().equals(dbfDataType)) {
log.error("uuid {}. Field {} from source {} has wrong data type (actual {}, expected {}).",
taskUuid,
currFieldName,
currTable,
dbfDataType != null ? dbfDataType.name() : "null",
currColumnStructure.getType());
valid = false;
continue;
}
int fieldLength = currField.getLength();
if (fieldLength != currColumnStructure.getLength()) {
if (!dbfDataType.equals(DBFDataType.DATE)) {
log.error("uuid {}. Field {} from source {} has wrong length (actual {}, expected {}).",
taskUuid,
currFieldName,
currTable,
fieldLength,
currColumnStructure.getLength());
valid = false;
continue;
} else {
if (fieldLength > currColumnStructure.getLength()) {
log.error("uuid {}. Field {} with DATE type has wrong length (length from file > length db field)",
taskUuid,
currFieldName);
valid = false;
continue;
}
}
}
int decimalCount = currField.getDecimalCount();
if (decimalCount != currColumnStructure.getDecimalDigits()) {
log.error("uuid {}. Field {} from source {} has wrong decimal count (actual {}, expected {}).",
taskUuid,
currFieldName,
currTable,
decimalCount,
currColumnStructure.getDecimalDigits());
valid = false;
}
if (!valid) {
return StageResult.ERROR;
}
}
} catch (IOException e) {
log.error(String.format("uuid %s. Can't read source for table %s.", taskUuid, currTable), e);
return StageResult.ERROR;
}
return StageResult.OK;
}
/**
* 1. Собирает из БД названия столбцов для дальнейшей валидации
* 2. Проверяет кодировку из настройки dbf.encoding-source
* 1. Проверяет кодировку из настройки dbf.encoding-source
*/
@Override
public void afterPropertiesSet() throws Exception {
initDBStructure();
initDBFCharset();
}
private void initDBStructure() throws MetaDataAccessException {
dbStructureDBF = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()),
dbMeta -> {
DBStructureDBF dbStructureDBF = new DBStructureDBF();
List<String> tableNamesFromResultSet = new LinkedList<>();
ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"});
while (tableNamesRS.next()) {
tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME"));
}
for (String tableNameFromResultSet : tableNamesFromResultSet) {
Table currTable = Table.tableForName(tableNameFromResultSet);
if (currTable == null) continue;
TableStructureDBF tableStructureDBF = new TableStructureDBF();
ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null);
while (columnNamesRS.next()) {
ColumnStructureDBF columnStructureDBF = new ColumnStructureDBF();
columnStructureDBF.setName(columnNamesRS.getString("COLUMN_NAME"));
columnStructureDBF.setType(ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE")));
columnStructureDBF.setLength(columnNamesRS.getInt("COLUMN_SIZE"));
columnStructureDBF.setDecimalDigits(columnNamesRS.getInt("DECIMAL_DIGITS"));
columnStructureDBF.setComment(columnNamesRS.getString("REMARKS"));
tableStructureDBF.getColumns().put(columnStructureDBF.getName(), columnStructureDBF);
}
assert !dbStructureDBF.getTables().containsKey(currTable);
dbStructureDBF.getTables().put(currTable, tableStructureDBF);
}
return dbStructureDBF;
});
}
private void initDBFCharset() {
try {
dbfCharset = Charset.forName(properties.getDbfEncoding());
@ -196,5 +41,4 @@ public class ValidateFields extends Stage implements InitializingBean {
throw new ConfigException("Unknown encoding from properties: " + properties.getDbfEncoding(), e);
}
}
}

8
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java
View file

@ -6,7 +6,7 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.dbf.importer.logic.stages.Processor;
import java.io.File;
@ -30,9 +30,9 @@ public class DBFImporterService {
@Scheduled(cron = "${dbf.check-src-dir-cron}")
public void run() {
Map<Table, List<File>> newFiles = fileChecker.checkNewFiles();
for (Map.Entry<Table, List<File>> newFilesEntry : newFiles.entrySet()) {
Table currTable = newFilesEntry.getKey();
Map<ETable, List<File>> newFiles = fileChecker.checkNewFiles();
for (Map.Entry<ETable, List<File>> newFilesEntry : newFiles.entrySet()) {
ETable currTable = newFilesEntry.getKey();
List<File> fileList = newFilesEntry.getValue();
for (File dbfFile : fileList) {
executorService.execute(() -> processor.process(ResultContainer.createNewTask(currTable, dbfFile)));

8
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java
View file

@ -1,7 +1,7 @@
package ru.spcex.clearing.dbf.importer.services;
import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.dbf.importer.properties.AProperties;
import java.io.File;
@ -15,8 +15,8 @@ public class FileChecker {
this.properties = properties;
}
public Map<Table, List<File>> checkNewFiles() {
Map<Table, List<File>> newFiles = new EnumMap<>(Table.class);
public Map<ETable, List<File>> checkNewFiles() {
Map<ETable, List<File>> newFiles = new EnumMap<>(ETable.class);
String srcDir = properties.getSrcDir();
List<File> dbfFiles = lsDBF(srcDir);
@ -25,7 +25,7 @@ public class FileChecker {
for (File dbfFile : dbfFiles) {
if (dbfFile.isDirectory()) continue;
Table currTable = Table.getTableForFilename(dbfFile.getName());
ETable currTable = ETable.getTableForFilename(dbfFile.getName());
if (currTable == null) continue;
List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());

23
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/utils/Const.java Normal file
View file

@ -0,0 +1,23 @@
package ru.spcex.clearing.dbf.importer.utils;
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
import ru.spcex.clearing.dbf.importer.logic.data.tables.*;
import java.util.HashMap;
import java.util.Map;
public class Const {
public static final Map<ETable, AbstractTable> MAPPING_ENUM_TABLE_OBJECT_TABLE = new HashMap<>();
static {
MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_01, new SDf01Table());
MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_02, new SDf02Table());
MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_08, new SDf08Table());
MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_12, new SDf12Table());
MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_16, new SDf16Table());
MAPPING_ENUM_TABLE_OBJECT_TABLE.put(ETable.DF_18, new SDf18Table());
}
}

9
clearing-parent/dbf-importer/src/main/resources/application.properties
View file

@ -1,17 +1,16 @@
server.port=8080
server.servlet.context-path=/importer
spring.main.web-application-type=servlet
db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres
db.driver=org.postgresql.Driver
db.login=clearing
db.password=Aa111111
dbf.check-src-dir-cron=* * * * 1 ?
dbf.encoding-source=cp866
dbf.insert-batch-size=100
dbf.delete-src-files=false
dbf.src-dir=D:\\dbf\\
dbf.threads-count=10
dbf.threads-count=10
import-dbf-service.hazelcast.cluster-members=127.0.0.1
import-dbf-service.hazelcast.login=dev
import-dbf-service.hazelcast.password=dev-pass

8
platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java
View file

@ -13,6 +13,7 @@ import ru.spcex.platform.imdg.api.Imdg;
import ru.spcex.platform.utils.log.ExceptionUtils;
import java.util.*;
public class ImdgHazelcast<T extends SpcexObjectBase> implements Imdg<T> {
private final Logger log = LoggerFactory.getLogger(getClass());
private IdGenerator idGenerator;
@ -109,7 +110,7 @@ public class ImdgHazelcast<T extends SpcexObjectBase> implements Imdg<T> {
Set<Long> ids = map.keySet(or);
Iterator<Long> idIterator = ids.iterator();
Collection<T> searchResult = new ArrayList<>();
while(idIterator.hasNext()) {
while (idIterator.hasNext()) {
T element = map.get(idIterator.next());
if (element != null) {
searchResult.add(element);
@ -118,6 +119,11 @@ public class ImdgHazelcast<T extends SpcexObjectBase> implements Imdg<T> {
return searchResult;
}
@Override
public Long nextIDSequenceFor() {
return idGenerator.newId();
}
public IMap<Long, T> getMap() {
return map;
}

7
platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgTransactionalHazelcast.java
View file

@ -105,7 +105,7 @@ public class ImdgTransactionalHazelcast<T extends SpcexObjectBase> implements Im
Set<Long> ids = map.keySet(or);
Iterator<Long> idIterator = ids.iterator();
Collection<T> searchResult = new ArrayList<>();
while(idIterator.hasNext()) {
while (idIterator.hasNext()) {
T element = map.get(idIterator.next());
if (element != null) {
searchResult.add(element);
@ -145,4 +145,9 @@ public class ImdgTransactionalHazelcast<T extends SpcexObjectBase> implements Im
public void setMapName(String mapName) {
this.mapName = mapName;
}
@Override
public Long nextIDSequenceFor() {
return idGenerator.newId();
}
}

15
platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/time/TimeUtil.java
View file

@ -4,8 +4,10 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
public class TimeUtil {
public static final DateTimeFormatter PROPERTY_DATE_FORMATTER = DateTimeFormatter.ofPattern("dd.MM.yyyy");
public static ZoneId zone = ZoneId.systemDefault();
/**
@ -27,4 +29,17 @@ public class TimeUtil {
public static Instant localDateToInstant(LocalDate date) {
return date.atStartOfDay(zone).toInstant();
}
public static LocalDate strToLocalDate(String date) {
try {
LocalDate parsed = LocalDate.parse(date, PROPERTY_DATE_FORMATTER);
return parsed;
} catch (Throwable e) {
return null;
}
}
public static Instant strToInstant(String date) {
return localDateToInstant(Objects.requireNonNull(strToLocalDate(date)));
}
}