treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dbf-exporter. jdbc -> hazelcast, ready for test

Browse source
This commit is contained in:
akulikov 2022-09-21 12:26:46 +03:00
parent f72d7ff8e2
commit 4e08b8acd0
19 changed files with 538 additions and 253 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
clearing-parent/dbf-exporter/pom.xml
View file

@ -30,26 +30,22 @@
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<!-- JDBC -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</dependency>
<dependency>
<groupId>com.mchange</groupId>
<artifactId>c3p0</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<!-- DBF files -->
<dependency>
<groupId>com.github.albfernandez</groupId>
<artifactId>javadbf</artifactId>
</dependency>
<!-- Own dependencies -->
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-imdg-api-hazelcast-impl</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>classes</artifactId>
</dependency>
</dependencies>
<build>

51
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java
View file

@ -1,21 +1,20 @@
package ru.spcex.clearing.dbf.exporter.config;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import ru.spcex.clearing.dbf.exporter.logic.stages.ExportFromDB;
import ru.spcex.clearing.dbf.exporter.logic.stages.ExportFromHazelcast;
import ru.spcex.clearing.dbf.exporter.logic.stages.PrepareDBFFile;
import ru.spcex.clearing.dbf.exporter.logic.stages.Stage;
import ru.spcex.clearing.dbf.exporter.properties.AProperties;
import ru.spcex.platform.imdg.api.ImdgProvider;
import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import javax.sql.DataSource;
import java.beans.PropertyVetoException;
import java.util.LinkedList;
import java.util.List;
@ -31,19 +30,26 @@ public class DBFExporterConfig {
this.context = context;
}
@Bean("dbfDataSource")
public DataSource dataSource() throws PropertyVetoException {
ComboPooledDataSource dataSource = new ComboPooledDataSource();
dataSource.setDriverClass(properties.getDbDriver());
dataSource.setJdbcUrl(properties.getJdbcUrl());
dataSource.setUser(properties.getDbLogin());
dataSource.setPassword(properties.getDbPassword());
return dataSource;
@Bean("taskExecutorHazelcastClientInitializer")
public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() {
return createThreadPoolTaskExecutor(1, true);
}
@Bean("dbfJdbcTemplate")
public JdbcTemplate jdbcTemplate(@Qualifier("dbfDataSource") DataSource dataSource) {
return new JdbcTemplate(dataSource);
@Bean("taskExecutorIdGeneratorAwaiter")
public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() {
return createThreadPoolTaskExecutor(1, false);
}
@Bean("imdgProvider")
public ImdgProvider imdgProvider(@Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
@Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
AProperties properties) {
HazelcastClientParams params = new HazelcastClientParams();
params.setClusterMembers(properties.getHazelcastClusterMembers());
params.setLogin(properties.getHazelcastLogin());
params.setPassword(properties.getHazelcastPassword());
params.setInstanceName("dbf-exporter");
return new HazelcastService(taskExecutorHazelcastClientInitializer, taskExecutorIdGeneratorAwaiter, params);
}
@Bean("pipeline")
@ -51,7 +57,7 @@ public class DBFExporterConfig {
List<Stage> pipeline = new LinkedList<>();
pipeline.add(context.getBean(PrepareDBFFile.class));
pipeline.add(context.getBean(ExportFromDB.class));
pipeline.add(context.getBean(ExportFromHazelcast.class));
return pipeline;
}
@ -68,4 +74,15 @@ public class DBFExporterConfig {
return executor;
}
private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) {
ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
if (maxPoolSz > 2) {
pool.setKeepAliveSeconds(60);
pool.setAllowCoreThreadTimeOut(true);
}
pool.setCorePoolSize(maxPoolSz);
pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion);
return pool;
}
}

4
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/controller/DefaultController.java
View file

@ -10,7 +10,7 @@ import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import ru.spcex.clearing.dbf.exporter.logic.data.ISqlFilter;
import ru.spcex.clearing.dbf.exporter.logic.data.IFilter;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table;
import ru.spcex.clearing.dbf.exporter.services.DBFExportService;
@ -37,7 +37,7 @@ public class DefaultController implements InitializingBean {
@ResponseBody
public String exportTables() {
log.info("Call export method for exporter controller");
Map<Table, ISqlFilter> tablesForExport = new EnumMap<>(Table.class);
Map<Table, IFilter> tablesForExport = new EnumMap<>(Table.class);
for (Table table : Table.values()) tablesForExport.put(table, null);
dbfExportService.run(tablesForExport);
return "export done, see log";

4
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ISqlFilter.java → clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/IFilter.java
View file

@ -3,6 +3,6 @@ package ru.spcex.clearing.dbf.exporter.logic.data;
/**
* Фильтр записей для экспорта
*/
public interface ISqlFilter {
String getSqlCondition();
public interface IFilter {
}

8
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ResultContainer.java
View file

@ -11,12 +11,12 @@ import java.util.UUID;
public class ResultContainer {
private UUID uuid;
private Table tableForExport;
private ISqlFilter filter;
private IFilter filter;
private File fileForExport;
protected ResultContainer() {}
public static ResultContainer createNewTask(Table tableForExport, ISqlFilter filter) {
public static ResultContainer createNewTask(Table tableForExport, IFilter filter) {
ResultContainer container = new ResultContainer();
container.tableForExport = tableForExport;
container.filter = filter;
@ -32,11 +32,11 @@ public class ResultContainer {
this.tableForExport = tableForExport;
}
public ISqlFilter getFilter() {
public IFilter getFilter() {
return filter;
}
public void setFilter(ISqlFilter filter) {
public void setFilter(IFilter filter) {
this.filter = filter;
}

25
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/ColumnType.java
View file

@ -2,37 +2,38 @@ package ru.spcex.clearing.dbf.exporter.logic.data.enums;
import com.linuxense.javadbf.DBFDataType;
import java.sql.Types;
import java.math.BigDecimal;
import java.time.LocalDate;
/**
* Типы данных в таблицах.
* Ставит в соответствие типы PostGRE и DBF
*/
public enum ColumnType {
VARCHAR(DBFDataType.CHARACTER, Types.VARCHAR),
CHARACTER(DBFDataType.CHARACTER, Types.CHAR),
NUMERIC(DBFDataType.NUMERIC, Types.NUMERIC),
DATE(DBFDataType.DATE, Types.DATE);
VARCHAR(DBFDataType.CHARACTER, String.class),
CHARACTER(DBFDataType.CHARACTER, Character.class),
NUMERIC(DBFDataType.NUMERIC, BigDecimal.class),
DATE(DBFDataType.DATE, LocalDate.class);
private final DBFDataType dbfType;
private final int sqlType;
private final Class<?> javaType;
ColumnType(DBFDataType dbfType, int postgreSqlType) {
ColumnType(DBFDataType dbfType, Class<?> javaType) {
this.dbfType = dbfType;
this.sqlType = postgreSqlType;
this.javaType = javaType;
}
public DBFDataType getDbfType() {
return dbfType;
}
public int getSqlType() {
return sqlType;
public Class<?> getJavaType() {
return javaType;
}
public static ColumnType getForSQLType(int sqlType) {
public static ColumnType getForSQLType(Class<?> sqlType) {
for (ColumnType columnType : values()) {
if (columnType.sqlType == sqlType) return columnType;
if (columnType.javaType == sqlType) return columnType;
}
return null;
}

62
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/Table.java
View file

@ -1,39 +1,49 @@
package ru.spcex.clearing.dbf.exporter.logic.data.enums;
public enum Table {
DF_01("DF-01"),
DF_02("DF-02"),
DF_03("DF-03"),
DF_04("DF-04"),
DF_05("DF-05"),
DF_06("DF-06");
import ru.clearing.classes.statics.data.sdf.*;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import ru.spcex.platform.classes.base.SpcexObjectBase;
public enum Table {
S_DF01("DF-01", IMDGDistributedNames.Map_SDf01, SDf01.class),
S_DF02("DF-02", IMDGDistributedNames.Map_SDf02, SDf02.class),
S_DF08("DF-08", IMDGDistributedNames.Map_SDf08, SDf08.class),
S_DF12("DF-12", IMDGDistributedNames.Map_SDf12, SDf12.class),
S_DF16("DF-16", IMDGDistributedNames.Map_SDf16, SDf16.class),
S_DF18("DF-18", IMDGDistributedNames.Map_SDf18, SDf18.class);
/**
* Префикс имени файла для экспорта
*/
private final String filePrefix;
Table(String prefix) {
this.filePrefix = prefix;
/**
* Имя мапы hazelcast
*/
private final String hazelcastMapName;
/**
* Класс объекта
*/
private final Class<? extends SpcexObjectBase> entityClass;
Table(String filePrefix, String hazelcastMapName, Class<? extends SpcexObjectBase> entityClass) {
this.filePrefix = filePrefix;
this.hazelcastMapName = hazelcastMapName;
this.entityClass = entityClass;
}
public boolean fileForThisTable(String filename) {
return filename != null && filename.startsWith(filePrefix);
}
public static Table getTableForFilename(String filename) {
for (Table table : Table.values()) {
if (table.fileForThisTable(filename)) return table;
}
return null;
}
public static Table tableForName(String name) {
for (Table table : values()) {
if (name.equalsIgnoreCase(table.name()))
return table;
}
return null;
public String getHazelcastMapName() {
return hazelcastMapName;
}
public String getFilePrefix() {
return filePrefix;
}
public Class<? extends SpcexObjectBase> getEntityClass() {
return entityClass;
}
}

139
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromDB.java
View file

@ -1,139 +0,0 @@
package ru.spcex.clearing.dbf.exporter.logic.stages;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import com.linuxense.javadbf.DBFWriter;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.JdbcUtils;
import org.springframework.jdbc.support.MetaDataAccessException;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.exporter.exceptions.ConfigException;
import ru.spcex.clearing.dbf.exporter.logic.data.ISqlFilter;
import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.ColumnType;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table;
import ru.spcex.clearing.dbf.exporter.properties.AProperties;
import java.io.File;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.sql.ResultSet;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.*;
/**
* Выгрузка данных из базы и их запись
*/
@Component
public class ExportFromDB extends Stage implements InitializingBean {
private final AProperties properties;
private final JdbcTemplate dbfJdbcTemplate;
private final Map<Table, DBFField[]> dbfFieldsForTable = new HashMap<>();
private Charset dbfCharset;
public ExportFromDB(@Qualifier("dbfExporterProperties") AProperties properties,
@Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) {
this.properties = properties;
this.dbfJdbcTemplate = dbfJdbcTemplate;
}
@Override
public StageResult process(ResultContainer resultContainer) {
Objects.requireNonNull(resultContainer.getTableForExport());
Objects.requireNonNull(resultContainer.getFileForExport());
Table table = resultContainer.getTableForExport();
ISqlFilter sqlFilter = resultContainer.getFilter();
String sqlCondition = sqlFilter != null ? " where " + sqlFilter : "";
String sql = "select * from " + table + sqlCondition;
List<Map<String, Object>> recordsFromDB = dbfJdbcTemplate.queryForList(sql);
File dbfFile = resultContainer.getFileForExport();
DBFField[] dbfFields = dbfFieldsForTable.get(table);
try (DBFWriter dbfWriter = new DBFWriter(dbfFile, dbfCharset)) {
dbfWriter.setFields(dbfFields);
for (Map<String, Object> record : recordsFromDB) {
int columnCount = dbfFields.length;
Object[] values = new Object[columnCount];
for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) {
DBFDataType currDBFType = dbfFields[columnIdx].getType();
Object currColumn = record.get(dbfFields[columnIdx].getName());
values[columnIdx] = convertToDBFValue(currDBFType, currColumn);
}
dbfWriter.addRecord(values);
}
} catch (Exception e) {
log.error(String.format("uuid %s. Can't export table %s to file %s. Table was skipped.", resultContainer.getUuid(), table, dbfFile), e);
return StageResult.ERROR;
}
return StageResult.COMPLETE;
}
private Object convertToDBFValue(DBFDataType dbfType, Object valueFromDB) throws Exception {
if (dbfType == DBFDataType.LOGICAL) {
return Boolean.parseBoolean(String.valueOf(valueFromDB));
} else if (dbfType == DBFDataType.DATE) {
LocalDate date = LocalDate.parse(String.valueOf(valueFromDB));
return new Date(date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli());
} else if (dbfType == DBFDataType.NUMERIC || dbfType == DBFDataType.FLOATING_POINT) {
return new BigDecimal(String.valueOf(valueFromDB));
} else {
return String.valueOf(valueFromDB);
}
}
/**
* 1. Собирает структуру БД для дальнейшей записи DBF файлов
* 2. Проверяет кодировку из настройки dbf.encoding
*/
@Override
public void afterPropertiesSet() throws Exception {
initDBStructure();
initDBFCharset();
}
private void initDBStructure() throws MetaDataAccessException {
boolean ok = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()),
dbMeta -> {
List<String> tableNamesFromResultSet = new LinkedList<>();
ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"});
while (tableNamesRS.next()) {
tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME"));
}
for (String tableNameFromResultSet : tableNamesFromResultSet) {
Table currTable = Table.tableForName(tableNameFromResultSet);
if (currTable == null) continue;
List<DBFField> dbfFields = new ArrayList<>();
ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null);
while (columnNamesRS.next()) {
String name = columnNamesRS.getString("COLUMN_NAME");
ColumnType type = ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE"));
if (type == null) throw new ConfigException("Unsupported ColumnType from DB");
int length = columnNamesRS.getInt("COLUMN_SIZE");
if (type == ColumnType.DATE) length = 8;
int decimalCount = columnNamesRS.getInt("DECIMAL_DIGITS");
DBFField dbfField = new DBFField(name.toUpperCase(), type.getDbfType(), length, decimalCount);
dbfFields.add(dbfField);
}
dbfFieldsForTable.put(currTable, dbfFields.toArray(DBFField[]::new));
}
return true;
});
}
private void initDBFCharset() {
try {
dbfCharset = Charset.forName(properties.getDbfEncoding());
} catch (Exception e) {
throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
}
}
}

129
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromHazelcast.java Normal file
View file

@ -0,0 +1,129 @@
package ru.spcex.clearing.dbf.exporter.logic.stages;
import com.linuxense.javadbf.DBFField;
import com.linuxense.javadbf.DBFWriter;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.clearing.classes.statics.data.sdf.*;
import ru.spcex.clearing.dbf.exporter.exceptions.ConfigException;
import ru.spcex.clearing.dbf.exporter.logic.data.IFilter;
import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table;
import ru.spcex.clearing.dbf.exporter.properties.AProperties;
import ru.spcex.clearing.dbf.exporter.services.converters.*;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.imdg.api.Imdg;
import ru.spcex.platform.imdg.api.ImdgProvider;
import java.io.File;
import java.nio.charset.Charset;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
/**
* Выгрузка данных из мапы hazelcast и их запись в файлы
*/
@Component
public class ExportFromHazelcast extends Stage implements InitializingBean {
private final AProperties properties;
private final ImdgProvider imdgProvider;
private final S_DF01_Converter s_df01_converter;
private final S_DF02_Converter s_df02_converter;
private final S_DF08_Converter s_df08_converter;
private final S_DF12_Converter s_df12_converter;
private final S_DF16_Converter s_df16_converter;
private final S_DF18_Converter s_df18_converter;
private final Map<Table, DBFField[]> dbfFieldsForTable = new HashMap<>();
private Charset dbfCharset;
public ExportFromHazelcast(@Qualifier("dbfExporterProperties") AProperties properties,
@Qualifier("imdgProvider") ImdgProvider imdgProvider,
S_DF01_Converter s_df01_converter,
S_DF02_Converter s_df02_converter,
S_DF08_Converter s_df08_converter,
S_DF12_Converter s_df12_converter,
S_DF16_Converter s_df16_converter,
S_DF18_Converter s_df18_converter) {
this.properties = properties;
this.imdgProvider = imdgProvider;
this.s_df01_converter = s_df01_converter;
this.s_df02_converter = s_df02_converter;
this.s_df08_converter = s_df08_converter;
this.s_df12_converter = s_df12_converter;
this.s_df16_converter = s_df16_converter;
this.s_df18_converter = s_df18_converter;
}
@Override
public StageResult process(ResultContainer resultContainer) {
Objects.requireNonNull(resultContainer.getTableForExport());
Objects.requireNonNull(resultContainer.getFileForExport());
Table table = resultContainer.getTableForExport();
IFilter filter = resultContainer.getFilter();
Imdg<? extends SpcexObjectBase> map = imdgProvider.getImdg(table.getHazelcastMapName(), table.getEntityClass());
File dbfFile = resultContainer.getFileForExport();
try (DBFWriter dbfWriter = new DBFWriter(dbfFile, dbfCharset)) {
dbfWriter.setFields(dbfFieldsForTable.get(table));
Collection<? extends SpcexObjectBase> allValues = map.getAllValues();
if (allValues.isEmpty()) {
log.info("uuid {}. Map {} is empty.", resultContainer.getUuid(), resultContainer.getTableForExport().getHazelcastMapName());
return StageResult.COMPLETE;
}
for (SpcexObjectBase value : allValues) {
Object[] values;
if (value instanceof SDf01 sdf01Value) values = s_df01_converter.toObjectArray(sdf01Value);
else if (value instanceof SDf02 sDf02Value) values = s_df02_converter.toObjectArray(sDf02Value);
else if (value instanceof SDf08 sDf08Value) values = s_df08_converter.toObjectArray(sDf08Value);
else if (value instanceof SDf12 sDf12Value) values = s_df12_converter.toObjectArray(sDf12Value);
else if (value instanceof SDf16 sDf16Value) values = s_df16_converter.toObjectArray(sDf16Value);
else if (value instanceof SDf18 sDf18Value) values = s_df18_converter.toObjectArray(sDf18Value);
else throw new Exception("Get unknown object from imdg. Class: " + value.getClass().getSimpleName());
dbfWriter.addRecord(values);
}
} catch (Exception e) {
log.error(String.format("uuid %s. Can't export table %s to file %s. Table was skipped.", resultContainer.getUuid(), table, dbfFile), e);
return StageResult.ERROR;
}
return StageResult.COMPLETE;
}
/**
* 1. Конструирует структуру DBF файлов для дальнейшей записи
* 2. Проверяет кодировку из настройки dbf.encoding
*/
@Override
public void afterPropertiesSet() {
initExportFileStructure();
initDBFCharset();
}
/**
* Инициализация структуры выходных файлов (заголовки столбцов)
*/
private void initExportFileStructure() {
dbfFieldsForTable.put(Table.S_DF01, s_df01_converter.getDBFHeaders());
dbfFieldsForTable.put(Table.S_DF02, s_df02_converter.getDBFHeaders());
dbfFieldsForTable.put(Table.S_DF08, s_df08_converter.getDBFHeaders());
dbfFieldsForTable.put(Table.S_DF12, s_df12_converter.getDBFHeaders());
dbfFieldsForTable.put(Table.S_DF16, s_df16_converter.getDBFHeaders());
dbfFieldsForTable.put(Table.S_DF18, s_df18_converter.getDBFHeaders());
}
private void initDBFCharset() {
try {
dbfCharset = Charset.forName(properties.getDbfEncoding());
} catch (Exception e) {
throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
}
}
}

47
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/properties/AProperties.java
View file

@ -6,17 +6,14 @@ import org.springframework.stereotype.Component;
@Component("dbfExporterProperties")
public class AProperties {
@Value("${db.jdbc-url}")
private String jdbcUrl;
@Value("${hazelcast.cluster-members}")
private String hazelcastClusterMembers;
@Value("${db.driver}")
private String dbDriver;
@Value("${hazelcast.login}")
private String hazelcastLogin;
@Value("${db.login}")
private String dbLogin;
@Value("${db.password}")
private String dbPassword;
@Value("${hazelcast.password}")
private String hazelcastPassword;
@Value("${dbf.out-dir}")
private String outDir;
@ -27,36 +24,28 @@ public class AProperties {
@Value("${dbf.threads-count}")
private int threadsCount;
public String getJdbcUrl() {
return jdbcUrl;
public String getHazelcastClusterMembers() {
return hazelcastClusterMembers;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
public void setHazelcastClusterMembers(String hazelcastClusterMembers) {
this.hazelcastClusterMembers = hazelcastClusterMembers;
}
public String getDbLogin() {
return dbLogin;
public String getHazelcastLogin() {
return hazelcastLogin;
}
public void setDbLogin(String dbLogin) {
this.dbLogin = dbLogin;
public void setHazelcastLogin(String hazelcastLogin) {
this.hazelcastLogin = hazelcastLogin;
}
public String getDbPassword() {
return dbPassword;
public String getHazelcastPassword() {
return hazelcastPassword;
}
public void setDbPassword(String dbPassword) {
this.dbPassword = dbPassword;
}
public String getDbDriver() {
return dbDriver;
}
public void setDbDriver(String dbDriver) {
this.dbDriver = dbDriver;
public void setHazelcastPassword(String hazelcastPassword) {
this.hazelcastPassword = hazelcastPassword;
}
public String getOutDir() {

8
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java
View file

@ -3,7 +3,7 @@ package ru.spcex.clearing.dbf.exporter.services;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.exporter.logic.data.ISqlFilter;
import ru.spcex.clearing.dbf.exporter.logic.data.IFilter;
import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table;
import ru.spcex.clearing.dbf.exporter.logic.stages.Processor;
@ -21,10 +21,10 @@ public class DBFExportService {
this.processor = processor;
}
public void run(Map<Table, ISqlFilter> tablesForExport) {
for (Map.Entry<Table, ISqlFilter> tableForExport : tablesForExport.entrySet()) {
public void run(Map<Table, IFilter> tablesForExport) {
for (Map.Entry<Table, IFilter> tableForExport : tablesForExport.entrySet()) {
Table table = tableForExport.getKey();
ISqlFilter filter = tableForExport.getValue();
IFilter filter = tableForExport.getValue();
executor.submit(() -> processor.process(ResultContainer.createNewTask(table, filter)));
}
}

25
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/DFConverter.java Normal file
View file

@ -0,0 +1,25 @@
package ru.spcex.clearing.dbf.exporter.services.converters;
import com.linuxense.javadbf.DBFField;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import java.sql.Date;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDate;
public abstract class DFConverter<T extends SpcexObjectBase> {
public abstract Object[] toObjectArray(T entity);
public abstract DBFField[] getDBFHeaders();
/**
* Приведение типов в соответствие (для записи)
* @param src исходный объект
* @return объект другого типа, если тип не поддерживается dbfWriter
*/
protected Object typeMatch(Object src) {
if (src instanceof Instant srcInstant) return Timestamp.from(srcInstant);
if (src instanceof LocalDate srcLocalDate) return Date.valueOf(srcLocalDate);
return src;
}
}

49
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF01_Converter.java Normal file
View file

@ -0,0 +1,49 @@
package ru.spcex.clearing.dbf.exporter.services.converters;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import org.springframework.stereotype.Service;
import ru.clearing.classes.statics.data.sdf.SDf01;
import java.util.LinkedList;
import java.util.List;
@Service
public class S_DF01_Converter extends DFConverter<SDf01> {
@Override
public Object[] toObjectArray(SDf01 entity) {
List<Object> values = new LinkedList<>();
values.add(typeMatch(entity.getCurr_code()));
values.add(typeMatch(entity.getAccount()));
values.add(typeMatch(entity.getRemainder()));
values.add(typeMatch(entity.getDeal()));
values.add(typeMatch(entity.getAcc_code()));
values.add(typeMatch(entity.getDat()));
values.add(typeMatch(entity.getMarket()));
values.add(typeMatch(entity.getAcc_name()));
values.add(typeMatch(entity.getAcc_type()));
values.add(typeMatch(entity.getSumengage()));
values.add(typeMatch(entity.getSumunblock()));
values.add(typeMatch(entity.getFile_type()));
return values.toArray(Object[]::new);
}
@Override
public DBFField[] getDBFHeaders() {
List<DBFField> dbfFields = new LinkedList<>();
dbfFields.add(new DBFField("curr_code", DBFDataType.CHARACTER, 12));
dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 35));
dbfFields.add(new DBFField("reminder", DBFDataType.CHARACTER, 22));
dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 10));
dbfFields.add(new DBFField("acc_code", DBFDataType.CHARACTER, 5));
dbfFields.add(new DBFField("dat", DBFDataType.CHARACTER, 8));
dbfFields.add(new DBFField("market", DBFDataType.CHARACTER, 1));
dbfFields.add(new DBFField("acc_name", DBFDataType.CHARACTER, 30));
dbfFields.add(new DBFField("acc_type", DBFDataType.CHARACTER, 2));
dbfFields.add(new DBFField("sumengage", DBFDataType.CHARACTER, 22));
dbfFields.add(new DBFField("sumunblock", DBFDataType.CHARACTER, 22));
dbfFields.add(new DBFField("file_type", DBFDataType.CHARACTER, 22));
return dbfFields.toArray(DBFField[]::new);
}
}

57
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF02_Converter.java Normal file
View file

@ -0,0 +1,57 @@
package ru.spcex.clearing.dbf.exporter.services.converters;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import org.springframework.stereotype.Service;
import ru.clearing.classes.statics.data.sdf.SDf02;
import java.util.LinkedList;
import java.util.List;
@Service
public class S_DF02_Converter extends DFConverter<SDf02> {
@Override
public Object[] toObjectArray(SDf02 entity) {
List<Object> values = new LinkedList<>();
values.add(typeMatch(entity.getCurr_code()));
values.add(typeMatch(entity.getAccount()));
values.add(typeMatch(entity.getRemainder()));
values.add(typeMatch(entity.getDeal()));
values.add(typeMatch(entity.getAcc_code()));
values.add(typeMatch(entity.getDat()));
values.add(typeMatch(entity.getMarket()));
values.add(typeMatch(entity.getAcc_name()));
values.add(typeMatch(entity.getAcc_type()));
values.add(typeMatch(entity.getSumengage()));
values.add(typeMatch(entity.getSumunblock()));
values.add(typeMatch(entity.getFile_type()));
values.add(typeMatch(entity.getResult()));
values.add(typeMatch(entity.getGenerationTime()));
values.add(typeMatch(entity.getGenerationId()));
values.add(typeMatch(entity.getInSDf01Id()));
return values.toArray(Object[]::new);
}
@Override
public DBFField[] getDBFHeaders() {
List<DBFField> dbfFields = new LinkedList<>();
dbfFields.add(new DBFField("curr_code", DBFDataType.CHARACTER, 12));
dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 35));
dbfFields.add(new DBFField("reminder", DBFDataType.CHARACTER, 22));
dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 10));
dbfFields.add(new DBFField("acc_code", DBFDataType.CHARACTER, 5));
dbfFields.add(new DBFField("dat", DBFDataType.CHARACTER, 8));
dbfFields.add(new DBFField("market", DBFDataType.CHARACTER, 1));
dbfFields.add(new DBFField("acc_name", DBFDataType.CHARACTER, 30));
dbfFields.add(new DBFField("acc_type", DBFDataType.CHARACTER, 2));
dbfFields.add(new DBFField("sumengage", DBFDataType.CHARACTER, 22));
dbfFields.add(new DBFField("sumunblock", DBFDataType.CHARACTER, 22));
dbfFields.add(new DBFField("file_type", DBFDataType.CHARACTER, 22));
dbfFields.add(new DBFField("result", DBFDataType.CHARACTER, 3));
dbfFields.add(new DBFField("gen_time", DBFDataType.DATE));
dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC));
dbfFields.add(new DBFField("insdf01_id", DBFDataType.NUMERIC));
return dbfFields.toArray(DBFField[]::new);
}
}

33
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF08_Converter.java Normal file
View file

@ -0,0 +1,33 @@
package ru.spcex.clearing.dbf.exporter.services.converters;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import org.springframework.stereotype.Service;
import ru.clearing.classes.statics.data.sdf.SDf08;
import java.util.LinkedList;
import java.util.List;
@Service
public class S_DF08_Converter extends DFConverter<SDf08> {
@Override
public Object[] toObjectArray(SDf08 entity) {
List<Object> values = new LinkedList<>();
values.add(typeMatch(entity.getNumber()));
values.add(typeMatch(entity.getDatetime()));
values.add(typeMatch(entity.getGenerationTime()));
values.add(typeMatch(entity.getGenerationId()));
return values.toArray(Object[]::new);
}
@Override
public DBFField[] getDBFHeaders() {
List<DBFField> dbfFields = new LinkedList<>();
dbfFields.add(new DBFField("number", DBFDataType.NUMERIC, 32));
dbfFields.add(new DBFField("datetime", DBFDataType.DATE));
dbfFields.add(new DBFField("gen_time", DBFDataType.DATE));
dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC));
return dbfFields.toArray(DBFField[]::new);
}
}

37
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF12_Converter.java Normal file
View file

@ -0,0 +1,37 @@
package ru.spcex.clearing.dbf.exporter.services.converters;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import org.springframework.stereotype.Service;
import ru.clearing.classes.statics.data.sdf.SDf12;
import java.util.LinkedList;
import java.util.List;
@Service
public class S_DF12_Converter extends DFConverter<SDf12> {
@Override
public Object[] toObjectArray(SDf12 entity) {
List<Object> values = new LinkedList<>();
values.add(typeMatch(entity.getAccount()));
values.add(typeMatch(entity.getDeal()));
values.add(typeMatch(entity.getStatus()));
values.add(typeMatch(entity.getFileName()));
values.add(typeMatch(entity.getGenerationTime()));
values.add(typeMatch(entity.getGenerationId()));
return values.toArray(Object[]::new);
}
@Override
public DBFField[] getDBFHeaders() {
List<DBFField> dbfFields = new LinkedList<>();
dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 25));
dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 25));
dbfFields.add(new DBFField("status", DBFDataType.NUMERIC, 32));
dbfFields.add(new DBFField("file_name", DBFDataType.CHARACTER, 254));
dbfFields.add(new DBFField("gen_time", DBFDataType.DATE));
dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC));
return dbfFields.toArray(DBFField[]::new);
}
}

43
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF16_Converter.java Normal file
View file

@ -0,0 +1,43 @@
package ru.spcex.clearing.dbf.exporter.services.converters;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import org.springframework.stereotype.Service;
import ru.clearing.classes.statics.data.sdf.SDf16;
import java.util.LinkedList;
import java.util.List;
@Service
public class S_DF16_Converter extends DFConverter<SDf16> {
@Override
public Object[] toObjectArray(SDf16 entity) {
List<Object> values = new LinkedList<>();
values.add(typeMatch(entity.getDate()));
values.add(typeMatch(entity.getAccount()));
values.add(typeMatch(entity.getSum()));
values.add(typeMatch(entity.getMarket()));
values.add(typeMatch(entity.getType()));
values.add(typeMatch(entity.getNumber()));
values.add(typeMatch(entity.getFileName()));
values.add(typeMatch(entity.getGenerationTime()));
values.add(typeMatch(entity.getGenerationId()));
return values.toArray(Object[]::new);
}
@Override
public DBFField[] getDBFHeaders() {
List<DBFField> dbfFields = new LinkedList<>();
dbfFields.add(new DBFField("date", DBFDataType.DATE));
dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 20));
dbfFields.add(new DBFField("sum", DBFDataType.NUMERIC, 32));
dbfFields.add(new DBFField("market", DBFDataType.CHARACTER, 1));
dbfFields.add(new DBFField("type", DBFDataType.CHARACTER, 1));
dbfFields.add(new DBFField("number", DBFDataType.NUMERIC, 32));
dbfFields.add(new DBFField("file_name", DBFDataType.CHARACTER, 254));
dbfFields.add(new DBFField("gen_time", DBFDataType.DATE));
dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC));
return dbfFields.toArray(DBFField[]::new);
}
}

39
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF18_Converter.java Normal file
View file

@ -0,0 +1,39 @@
package ru.spcex.clearing.dbf.exporter.services.converters;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFField;
import org.springframework.stereotype.Service;
import ru.clearing.classes.statics.data.sdf.SDf18;
import java.util.LinkedList;
import java.util.List;
@Service
public class S_DF18_Converter extends DFConverter<SDf18> {
@Override
public Object[] toObjectArray(SDf18 entity) {
List<Object> values = new LinkedList<>();
values.add(typeMatch(entity.getAccount()));
values.add(typeMatch(entity.getDeal()));
values.add(typeMatch(entity.getStatus()));
values.add(typeMatch(entity.getResult()));
values.add(typeMatch(entity.getGenerationTime()));
values.add(typeMatch(entity.getGenerationId()));
values.add(typeMatch(entity.getInSDf12Id()));
return values.toArray(Object[]::new);
}
@Override
public DBFField[] getDBFHeaders() {
List<DBFField> dbfFields = new LinkedList<>();
dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 25));
dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 4));
dbfFields.add(new DBFField("status", DBFDataType.NUMERIC, 32));
dbfFields.add(new DBFField("result", DBFDataType.NUMERIC, 32));
dbfFields.add(new DBFField("gen_time", DBFDataType.DATE));
dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC));
dbfFields.add(new DBFField("insdf12_id", DBFDataType.NUMERIC));
return dbfFields.toArray(DBFField[]::new);
}
}

7
clearing-parent/dbf-exporter/src/main/resources/application.properties
View file

@ -2,10 +2,9 @@ server.port=8080
server.servlet.context-path=/exporter
spring.main.web-application-type=servlet
db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres
db.driver=org.postgresql.Driver
db.login=clearing
db.password=Aa111111
hazelcast.cluster-members=127.0.0.1:5701
hazelcast.login=dev
hazelcast.password=dev-pass
dbf.encoding=cp866
dbf.threads-count=10