diff --git a/clearing-parent/dbf-exporter/pom.xml b/clearing-parent/dbf-exporter/pom.xml index dc03fe6c8..3e4163572 100644 --- a/clearing-parent/dbf-exporter/pom.xml +++ b/clearing-parent/dbf-exporter/pom.xml @@ -30,26 +30,22 @@ spring-boot-autoconfigure - - - org.springframework - spring-jdbc - - - com.mchange - c3p0 - - - org.postgresql - postgresql - - com.github.albfernandez javadbf + + + ru.spcex.platform + platform-imdg-api-hazelcast-impl + + + ru.spcex.clearing + classes + + diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java index cbad9106a..811c2733d 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java @@ -1,21 +1,20 @@ package ru.spcex.clearing.dbf.exporter.config; -import com.mchange.v2.c3p0.ComboPooledDataSource; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import ru.spcex.clearing.dbf.exporter.logic.stages.ExportFromDB; +import ru.spcex.clearing.dbf.exporter.logic.stages.ExportFromHazelcast; import ru.spcex.clearing.dbf.exporter.logic.stages.PrepareDBFFile; import ru.spcex.clearing.dbf.exporter.logic.stages.Stage; import ru.spcex.clearing.dbf.exporter.properties.AProperties; +import ru.spcex.platform.imdg.api.ImdgProvider; +import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; -import javax.sql.DataSource; -import java.beans.PropertyVetoException; import java.util.LinkedList; import java.util.List; @@ -31,19 +30,26 @@ public class DBFExporterConfig { this.context = context; } - @Bean("dbfDataSource") - public DataSource dataSource() throws PropertyVetoException { - ComboPooledDataSource dataSource = new ComboPooledDataSource(); - dataSource.setDriverClass(properties.getDbDriver()); - dataSource.setJdbcUrl(properties.getJdbcUrl()); - dataSource.setUser(properties.getDbLogin()); - dataSource.setPassword(properties.getDbPassword()); - return dataSource; + @Bean("taskExecutorHazelcastClientInitializer") + public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() { + return createThreadPoolTaskExecutor(1, true); } - @Bean("dbfJdbcTemplate") - public JdbcTemplate jdbcTemplate(@Qualifier("dbfDataSource") DataSource dataSource) { - return new JdbcTemplate(dataSource); + @Bean("taskExecutorIdGeneratorAwaiter") + public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() { + return createThreadPoolTaskExecutor(1, false); + } + + @Bean("imdgProvider") + public ImdgProvider imdgProvider(@Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + AProperties properties) { + HazelcastClientParams params = new HazelcastClientParams(); + params.setClusterMembers(properties.getHazelcastClusterMembers()); + params.setLogin(properties.getHazelcastLogin()); + params.setPassword(properties.getHazelcastPassword()); + params.setInstanceName("dbf-exporter"); + return new HazelcastService(taskExecutorHazelcastClientInitializer, taskExecutorIdGeneratorAwaiter, params); } @Bean("pipeline") @@ -51,7 +57,7 @@ public class DBFExporterConfig { List pipeline = new LinkedList<>(); pipeline.add(context.getBean(PrepareDBFFile.class)); - pipeline.add(context.getBean(ExportFromDB.class)); + pipeline.add(context.getBean(ExportFromHazelcast.class)); return pipeline; } @@ -68,4 +74,15 @@ public class DBFExporterConfig { return executor; } + private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) { + ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor(); + if (maxPoolSz > 2) { + pool.setKeepAliveSeconds(60); + pool.setAllowCoreThreadTimeOut(true); + } + pool.setCorePoolSize(maxPoolSz); + pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion); + return pool; + } + } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/controller/DefaultController.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/controller/DefaultController.java index 5889d1604..0cd525aa0 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/controller/DefaultController.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/controller/DefaultController.java @@ -10,7 +10,7 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; -import ru.spcex.clearing.dbf.exporter.logic.data.ISqlFilter; +import ru.spcex.clearing.dbf.exporter.logic.data.IFilter; import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table; import ru.spcex.clearing.dbf.exporter.services.DBFExportService; @@ -37,7 +37,7 @@ public class DefaultController implements InitializingBean { @ResponseBody public String exportTables() { log.info("Call export method for exporter controller"); - Map tablesForExport = new EnumMap<>(Table.class); + Map tablesForExport = new EnumMap<>(Table.class); for (Table table : Table.values()) tablesForExport.put(table, null); dbfExportService.run(tablesForExport); return "export done, see log"; diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ISqlFilter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/IFilter.java similarity index 66% rename from clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ISqlFilter.java rename to clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/IFilter.java index 668a5e6f3..56c45b249 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ISqlFilter.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/IFilter.java @@ -3,6 +3,6 @@ package ru.spcex.clearing.dbf.exporter.logic.data; /** * Фильтр записей для экспорта */ -public interface ISqlFilter { - String getSqlCondition(); +public interface IFilter { + } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ResultContainer.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ResultContainer.java index ecdc1b43e..9513b0c4f 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ResultContainer.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/ResultContainer.java @@ -11,12 +11,12 @@ import java.util.UUID; public class ResultContainer { private UUID uuid; private Table tableForExport; - private ISqlFilter filter; + private IFilter filter; private File fileForExport; protected ResultContainer() {} - public static ResultContainer createNewTask(Table tableForExport, ISqlFilter filter) { + public static ResultContainer createNewTask(Table tableForExport, IFilter filter) { ResultContainer container = new ResultContainer(); container.tableForExport = tableForExport; container.filter = filter; @@ -32,11 +32,11 @@ public class ResultContainer { this.tableForExport = tableForExport; } - public ISqlFilter getFilter() { + public IFilter getFilter() { return filter; } - public void setFilter(ISqlFilter filter) { + public void setFilter(IFilter filter) { this.filter = filter; } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/ColumnType.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/ColumnType.java index fbf77e773..67fd51746 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/ColumnType.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/ColumnType.java @@ -2,37 +2,38 @@ package ru.spcex.clearing.dbf.exporter.logic.data.enums; import com.linuxense.javadbf.DBFDataType; -import java.sql.Types; +import java.math.BigDecimal; +import java.time.LocalDate; /** * Типы данных в таблицах. * Ставит в соответствие типы PostGRE и DBF */ public enum ColumnType { - VARCHAR(DBFDataType.CHARACTER, Types.VARCHAR), - CHARACTER(DBFDataType.CHARACTER, Types.CHAR), - NUMERIC(DBFDataType.NUMERIC, Types.NUMERIC), - DATE(DBFDataType.DATE, Types.DATE); + VARCHAR(DBFDataType.CHARACTER, String.class), + CHARACTER(DBFDataType.CHARACTER, Character.class), + NUMERIC(DBFDataType.NUMERIC, BigDecimal.class), + DATE(DBFDataType.DATE, LocalDate.class); private final DBFDataType dbfType; - private final int sqlType; + private final Class javaType; - ColumnType(DBFDataType dbfType, int postgreSqlType) { + ColumnType(DBFDataType dbfType, Class javaType) { this.dbfType = dbfType; - this.sqlType = postgreSqlType; + this.javaType = javaType; } public DBFDataType getDbfType() { return dbfType; } - public int getSqlType() { - return sqlType; + public Class getJavaType() { + return javaType; } - public static ColumnType getForSQLType(int sqlType) { + public static ColumnType getForSQLType(Class sqlType) { for (ColumnType columnType : values()) { - if (columnType.sqlType == sqlType) return columnType; + if (columnType.javaType == sqlType) return columnType; } return null; } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/Table.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/Table.java index 697490f7e..740ad040b 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/Table.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/data/enums/Table.java @@ -1,39 +1,49 @@ package ru.spcex.clearing.dbf.exporter.logic.data.enums; -public enum Table { - DF_01("DF-01"), - DF_02("DF-02"), - DF_03("DF-03"), - DF_04("DF-04"), - DF_05("DF-05"), - DF_06("DF-06"); +import ru.clearing.classes.statics.data.sdf.*; +import ru.spcex.clearing.imdg.IMDGDistributedNames; +import ru.spcex.platform.classes.base.SpcexObjectBase; +public enum Table { + S_DF01("DF-01", IMDGDistributedNames.Map_SDf01, SDf01.class), + S_DF02("DF-02", IMDGDistributedNames.Map_SDf02, SDf02.class), + S_DF08("DF-08", IMDGDistributedNames.Map_SDf08, SDf08.class), + S_DF12("DF-12", IMDGDistributedNames.Map_SDf12, SDf12.class), + S_DF16("DF-16", IMDGDistributedNames.Map_SDf16, SDf16.class), + S_DF18("DF-18", IMDGDistributedNames.Map_SDf18, SDf18.class); + + /** + * Префикс имени файла для экспорта + */ private final String filePrefix; - Table(String prefix) { - this.filePrefix = prefix; + /** + * Имя мапы hazelcast + */ + private final String hazelcastMapName; + + /** + * Класс объекта + */ + private final Class entityClass; + + + Table(String filePrefix, String hazelcastMapName, Class entityClass) { + this.filePrefix = filePrefix; + this.hazelcastMapName = hazelcastMapName; + this.entityClass = entityClass; } - public boolean fileForThisTable(String filename) { - return filename != null && filename.startsWith(filePrefix); - } - - public static Table getTableForFilename(String filename) { - for (Table table : Table.values()) { - if (table.fileForThisTable(filename)) return table; - } - return null; - } - - public static Table tableForName(String name) { - for (Table table : values()) { - if (name.equalsIgnoreCase(table.name())) - return table; - } - return null; + public String getHazelcastMapName() { + return hazelcastMapName; } public String getFilePrefix() { return filePrefix; } + + public Class getEntityClass() { + return entityClass; + } + } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromDB.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromDB.java deleted file mode 100644 index a95f588ca..000000000 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromDB.java +++ /dev/null @@ -1,139 +0,0 @@ -package ru.spcex.clearing.dbf.exporter.logic.stages; - -import com.linuxense.javadbf.DBFDataType; -import com.linuxense.javadbf.DBFField; -import com.linuxense.javadbf.DBFWriter; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.support.JdbcUtils; -import org.springframework.jdbc.support.MetaDataAccessException; -import org.springframework.stereotype.Component; -import ru.spcex.clearing.dbf.exporter.exceptions.ConfigException; -import ru.spcex.clearing.dbf.exporter.logic.data.ISqlFilter; -import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer; -import ru.spcex.clearing.dbf.exporter.logic.data.enums.ColumnType; -import ru.spcex.clearing.dbf.exporter.logic.data.enums.StageResult; -import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table; -import ru.spcex.clearing.dbf.exporter.properties.AProperties; - -import java.io.File; -import java.math.BigDecimal; -import java.nio.charset.Charset; -import java.sql.ResultSet; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.util.*; - -/** - * Выгрузка данных из базы и их запись - */ -@Component -public class ExportFromDB extends Stage implements InitializingBean { - private final AProperties properties; - private final JdbcTemplate dbfJdbcTemplate; - private final Map dbfFieldsForTable = new HashMap<>(); - - private Charset dbfCharset; - - public ExportFromDB(@Qualifier("dbfExporterProperties") AProperties properties, - @Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) { - this.properties = properties; - this.dbfJdbcTemplate = dbfJdbcTemplate; - } - - @Override - public StageResult process(ResultContainer resultContainer) { - Objects.requireNonNull(resultContainer.getTableForExport()); - Objects.requireNonNull(resultContainer.getFileForExport()); - - Table table = resultContainer.getTableForExport(); - ISqlFilter sqlFilter = resultContainer.getFilter(); - String sqlCondition = sqlFilter != null ? " where " + sqlFilter : ""; - - String sql = "select * from " + table + sqlCondition; - List> recordsFromDB = dbfJdbcTemplate.queryForList(sql); - - File dbfFile = resultContainer.getFileForExport(); - DBFField[] dbfFields = dbfFieldsForTable.get(table); - - try (DBFWriter dbfWriter = new DBFWriter(dbfFile, dbfCharset)) { - dbfWriter.setFields(dbfFields); - for (Map record : recordsFromDB) { - int columnCount = dbfFields.length; - Object[] values = new Object[columnCount]; - for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) { - DBFDataType currDBFType = dbfFields[columnIdx].getType(); - Object currColumn = record.get(dbfFields[columnIdx].getName()); - values[columnIdx] = convertToDBFValue(currDBFType, currColumn); - } - dbfWriter.addRecord(values); - } - } catch (Exception e) { - log.error(String.format("uuid %s. Can't export table %s to file %s. Table was skipped.", resultContainer.getUuid(), table, dbfFile), e); - return StageResult.ERROR; - } - - return StageResult.COMPLETE; - } - - private Object convertToDBFValue(DBFDataType dbfType, Object valueFromDB) throws Exception { - if (dbfType == DBFDataType.LOGICAL) { - return Boolean.parseBoolean(String.valueOf(valueFromDB)); - } else if (dbfType == DBFDataType.DATE) { - LocalDate date = LocalDate.parse(String.valueOf(valueFromDB)); - return new Date(date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli()); - } else if (dbfType == DBFDataType.NUMERIC || dbfType == DBFDataType.FLOATING_POINT) { - return new BigDecimal(String.valueOf(valueFromDB)); - } else { - return String.valueOf(valueFromDB); - } - } - - /** - * 1. Собирает структуру БД для дальнейшей записи DBF файлов - * 2. Проверяет кодировку из настройки dbf.encoding - */ - @Override - public void afterPropertiesSet() throws Exception { - initDBStructure(); - initDBFCharset(); - } - - private void initDBStructure() throws MetaDataAccessException { - boolean ok = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()), - dbMeta -> { - List tableNamesFromResultSet = new LinkedList<>(); - ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"}); - while (tableNamesRS.next()) { - tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME")); - } - for (String tableNameFromResultSet : tableNamesFromResultSet) { - Table currTable = Table.tableForName(tableNameFromResultSet); - if (currTable == null) continue; - List dbfFields = new ArrayList<>(); - ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null); - while (columnNamesRS.next()) { - String name = columnNamesRS.getString("COLUMN_NAME"); - ColumnType type = ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE")); - if (type == null) throw new ConfigException("Unsupported ColumnType from DB"); - int length = columnNamesRS.getInt("COLUMN_SIZE"); - if (type == ColumnType.DATE) length = 8; - int decimalCount = columnNamesRS.getInt("DECIMAL_DIGITS"); - DBFField dbfField = new DBFField(name.toUpperCase(), type.getDbfType(), length, decimalCount); - dbfFields.add(dbfField); - } - dbfFieldsForTable.put(currTable, dbfFields.toArray(DBFField[]::new)); - } - return true; - }); - } - - private void initDBFCharset() { - try { - dbfCharset = Charset.forName(properties.getDbfEncoding()); - } catch (Exception e) { - throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e); - } - } -} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromHazelcast.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromHazelcast.java new file mode 100644 index 000000000..72b67032a --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/ExportFromHazelcast.java @@ -0,0 +1,129 @@ +package ru.spcex.clearing.dbf.exporter.logic.stages; + +import com.linuxense.javadbf.DBFField; +import com.linuxense.javadbf.DBFWriter; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import ru.clearing.classes.statics.data.sdf.*; +import ru.spcex.clearing.dbf.exporter.exceptions.ConfigException; +import ru.spcex.clearing.dbf.exporter.logic.data.IFilter; +import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.exporter.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table; +import ru.spcex.clearing.dbf.exporter.properties.AProperties; +import ru.spcex.clearing.dbf.exporter.services.converters.*; +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.imdg.api.Imdg; +import ru.spcex.platform.imdg.api.ImdgProvider; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Выгрузка данных из мапы hazelcast и их запись в файлы + */ +@Component +public class ExportFromHazelcast extends Stage implements InitializingBean { + private final AProperties properties; + private final ImdgProvider imdgProvider; + + private final S_DF01_Converter s_df01_converter; + private final S_DF02_Converter s_df02_converter; + private final S_DF08_Converter s_df08_converter; + private final S_DF12_Converter s_df12_converter; + private final S_DF16_Converter s_df16_converter; + private final S_DF18_Converter s_df18_converter; + + private final Map dbfFieldsForTable = new HashMap<>(); + private Charset dbfCharset; + + public ExportFromHazelcast(@Qualifier("dbfExporterProperties") AProperties properties, + @Qualifier("imdgProvider") ImdgProvider imdgProvider, + S_DF01_Converter s_df01_converter, + S_DF02_Converter s_df02_converter, + S_DF08_Converter s_df08_converter, + S_DF12_Converter s_df12_converter, + S_DF16_Converter s_df16_converter, + S_DF18_Converter s_df18_converter) { + this.properties = properties; + this.imdgProvider = imdgProvider; + this.s_df01_converter = s_df01_converter; + this.s_df02_converter = s_df02_converter; + this.s_df08_converter = s_df08_converter; + this.s_df12_converter = s_df12_converter; + this.s_df16_converter = s_df16_converter; + this.s_df18_converter = s_df18_converter; + } + + @Override + public StageResult process(ResultContainer resultContainer) { + Objects.requireNonNull(resultContainer.getTableForExport()); + Objects.requireNonNull(resultContainer.getFileForExport()); + + Table table = resultContainer.getTableForExport(); + IFilter filter = resultContainer.getFilter(); + + Imdg map = imdgProvider.getImdg(table.getHazelcastMapName(), table.getEntityClass()); + File dbfFile = resultContainer.getFileForExport(); + try (DBFWriter dbfWriter = new DBFWriter(dbfFile, dbfCharset)) { + dbfWriter.setFields(dbfFieldsForTable.get(table)); + Collection allValues = map.getAllValues(); + if (allValues.isEmpty()) { + log.info("uuid {}. Map {} is empty.", resultContainer.getUuid(), resultContainer.getTableForExport().getHazelcastMapName()); + return StageResult.COMPLETE; + } + for (SpcexObjectBase value : allValues) { + Object[] values; + if (value instanceof SDf01 sdf01Value) values = s_df01_converter.toObjectArray(sdf01Value); + else if (value instanceof SDf02 sDf02Value) values = s_df02_converter.toObjectArray(sDf02Value); + else if (value instanceof SDf08 sDf08Value) values = s_df08_converter.toObjectArray(sDf08Value); + else if (value instanceof SDf12 sDf12Value) values = s_df12_converter.toObjectArray(sDf12Value); + else if (value instanceof SDf16 sDf16Value) values = s_df16_converter.toObjectArray(sDf16Value); + else if (value instanceof SDf18 sDf18Value) values = s_df18_converter.toObjectArray(sDf18Value); + else throw new Exception("Get unknown object from imdg. Class: " + value.getClass().getSimpleName()); + dbfWriter.addRecord(values); + } + } catch (Exception e) { + log.error(String.format("uuid %s. Can't export table %s to file %s. Table was skipped.", resultContainer.getUuid(), table, dbfFile), e); + return StageResult.ERROR; + } + + return StageResult.COMPLETE; + } + + /** + * 1. Конструирует структуру DBF файлов для дальнейшей записи + * 2. Проверяет кодировку из настройки dbf.encoding + */ + @Override + public void afterPropertiesSet() { + initExportFileStructure(); + initDBFCharset(); + } + + /** + * Инициализация структуры выходных файлов (заголовки столбцов) + */ + private void initExportFileStructure() { + dbfFieldsForTable.put(Table.S_DF01, s_df01_converter.getDBFHeaders()); + dbfFieldsForTable.put(Table.S_DF02, s_df02_converter.getDBFHeaders()); + dbfFieldsForTable.put(Table.S_DF08, s_df08_converter.getDBFHeaders()); + dbfFieldsForTable.put(Table.S_DF12, s_df12_converter.getDBFHeaders()); + dbfFieldsForTable.put(Table.S_DF16, s_df16_converter.getDBFHeaders()); + dbfFieldsForTable.put(Table.S_DF18, s_df18_converter.getDBFHeaders()); + } + + private void initDBFCharset() { + try { + dbfCharset = Charset.forName(properties.getDbfEncoding()); + } catch (Exception e) { + throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e); + } + } + +} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/properties/AProperties.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/properties/AProperties.java index 26e589067..46ee82324 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/properties/AProperties.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/properties/AProperties.java @@ -6,17 +6,14 @@ import org.springframework.stereotype.Component; @Component("dbfExporterProperties") public class AProperties { - @Value("${db.jdbc-url}") - private String jdbcUrl; + @Value("${hazelcast.cluster-members}") + private String hazelcastClusterMembers; - @Value("${db.driver}") - private String dbDriver; + @Value("${hazelcast.login}") + private String hazelcastLogin; - @Value("${db.login}") - private String dbLogin; - - @Value("${db.password}") - private String dbPassword; + @Value("${hazelcast.password}") + private String hazelcastPassword; @Value("${dbf.out-dir}") private String outDir; @@ -27,36 +24,28 @@ public class AProperties { @Value("${dbf.threads-count}") private int threadsCount; - public String getJdbcUrl() { - return jdbcUrl; + public String getHazelcastClusterMembers() { + return hazelcastClusterMembers; } - public void setJdbcUrl(String jdbcUrl) { - this.jdbcUrl = jdbcUrl; + public void setHazelcastClusterMembers(String hazelcastClusterMembers) { + this.hazelcastClusterMembers = hazelcastClusterMembers; } - public String getDbLogin() { - return dbLogin; + public String getHazelcastLogin() { + return hazelcastLogin; } - public void setDbLogin(String dbLogin) { - this.dbLogin = dbLogin; + public void setHazelcastLogin(String hazelcastLogin) { + this.hazelcastLogin = hazelcastLogin; } - public String getDbPassword() { - return dbPassword; + public String getHazelcastPassword() { + return hazelcastPassword; } - public void setDbPassword(String dbPassword) { - this.dbPassword = dbPassword; - } - - public String getDbDriver() { - return dbDriver; - } - - public void setDbDriver(String dbDriver) { - this.dbDriver = dbDriver; + public void setHazelcastPassword(String hazelcastPassword) { + this.hazelcastPassword = hazelcastPassword; } public String getOutDir() { diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java index 894be92e7..fd1bc1a8f 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java @@ -3,7 +3,7 @@ package ru.spcex.clearing.dbf.exporter.services; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; -import ru.spcex.clearing.dbf.exporter.logic.data.ISqlFilter; +import ru.spcex.clearing.dbf.exporter.logic.data.IFilter; import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer; import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table; import ru.spcex.clearing.dbf.exporter.logic.stages.Processor; @@ -21,10 +21,10 @@ public class DBFExportService { this.processor = processor; } - public void run(Map tablesForExport) { - for (Map.Entry tableForExport : tablesForExport.entrySet()) { + public void run(Map tablesForExport) { + for (Map.Entry tableForExport : tablesForExport.entrySet()) { Table table = tableForExport.getKey(); - ISqlFilter filter = tableForExport.getValue(); + IFilter filter = tableForExport.getValue(); executor.submit(() -> processor.process(ResultContainer.createNewTask(table, filter))); } } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/DFConverter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/DFConverter.java new file mode 100644 index 000000000..ddf25db63 --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/DFConverter.java @@ -0,0 +1,25 @@ +package ru.spcex.clearing.dbf.exporter.services.converters; + +import com.linuxense.javadbf.DBFField; +import ru.spcex.platform.classes.base.SpcexObjectBase; + +import java.sql.Date; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; + +public abstract class DFConverter { + public abstract Object[] toObjectArray(T entity); + public abstract DBFField[] getDBFHeaders(); + + /** + * Приведение типов в соответствие (для записи) + * @param src исходный объект + * @return объект другого типа, если тип не поддерживается dbfWriter + */ + protected Object typeMatch(Object src) { + if (src instanceof Instant srcInstant) return Timestamp.from(srcInstant); + if (src instanceof LocalDate srcLocalDate) return Date.valueOf(srcLocalDate); + return src; + } +} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF01_Converter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF01_Converter.java new file mode 100644 index 000000000..e404d4d9f --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF01_Converter.java @@ -0,0 +1,49 @@ +package ru.spcex.clearing.dbf.exporter.services.converters; + +import com.linuxense.javadbf.DBFDataType; +import com.linuxense.javadbf.DBFField; +import org.springframework.stereotype.Service; +import ru.clearing.classes.statics.data.sdf.SDf01; + +import java.util.LinkedList; +import java.util.List; + +@Service +public class S_DF01_Converter extends DFConverter { + @Override + public Object[] toObjectArray(SDf01 entity) { + List values = new LinkedList<>(); + values.add(typeMatch(entity.getCurr_code())); + values.add(typeMatch(entity.getAccount())); + values.add(typeMatch(entity.getRemainder())); + values.add(typeMatch(entity.getDeal())); + values.add(typeMatch(entity.getAcc_code())); + values.add(typeMatch(entity.getDat())); + values.add(typeMatch(entity.getMarket())); + values.add(typeMatch(entity.getAcc_name())); + values.add(typeMatch(entity.getAcc_type())); + values.add(typeMatch(entity.getSumengage())); + values.add(typeMatch(entity.getSumunblock())); + values.add(typeMatch(entity.getFile_type())); + return values.toArray(Object[]::new); + } + + @Override + public DBFField[] getDBFHeaders() { + List dbfFields = new LinkedList<>(); + dbfFields.add(new DBFField("curr_code", DBFDataType.CHARACTER, 12)); + dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 35)); + dbfFields.add(new DBFField("reminder", DBFDataType.CHARACTER, 22)); + dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 10)); + dbfFields.add(new DBFField("acc_code", DBFDataType.CHARACTER, 5)); + dbfFields.add(new DBFField("dat", DBFDataType.CHARACTER, 8)); + dbfFields.add(new DBFField("market", DBFDataType.CHARACTER, 1)); + dbfFields.add(new DBFField("acc_name", DBFDataType.CHARACTER, 30)); + dbfFields.add(new DBFField("acc_type", DBFDataType.CHARACTER, 2)); + dbfFields.add(new DBFField("sumengage", DBFDataType.CHARACTER, 22)); + dbfFields.add(new DBFField("sumunblock", DBFDataType.CHARACTER, 22)); + dbfFields.add(new DBFField("file_type", DBFDataType.CHARACTER, 22)); + return dbfFields.toArray(DBFField[]::new); + } + +} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF02_Converter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF02_Converter.java new file mode 100644 index 000000000..15f1e4e9f --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF02_Converter.java @@ -0,0 +1,57 @@ +package ru.spcex.clearing.dbf.exporter.services.converters; + +import com.linuxense.javadbf.DBFDataType; +import com.linuxense.javadbf.DBFField; +import org.springframework.stereotype.Service; +import ru.clearing.classes.statics.data.sdf.SDf02; + +import java.util.LinkedList; +import java.util.List; + +@Service +public class S_DF02_Converter extends DFConverter { + @Override + public Object[] toObjectArray(SDf02 entity) { + List values = new LinkedList<>(); + values.add(typeMatch(entity.getCurr_code())); + values.add(typeMatch(entity.getAccount())); + values.add(typeMatch(entity.getRemainder())); + values.add(typeMatch(entity.getDeal())); + values.add(typeMatch(entity.getAcc_code())); + values.add(typeMatch(entity.getDat())); + values.add(typeMatch(entity.getMarket())); + values.add(typeMatch(entity.getAcc_name())); + values.add(typeMatch(entity.getAcc_type())); + values.add(typeMatch(entity.getSumengage())); + values.add(typeMatch(entity.getSumunblock())); + values.add(typeMatch(entity.getFile_type())); + values.add(typeMatch(entity.getResult())); + values.add(typeMatch(entity.getGenerationTime())); + values.add(typeMatch(entity.getGenerationId())); + values.add(typeMatch(entity.getInSDf01Id())); + return values.toArray(Object[]::new); + } + + @Override + public DBFField[] getDBFHeaders() { + List dbfFields = new LinkedList<>(); + dbfFields.add(new DBFField("curr_code", DBFDataType.CHARACTER, 12)); + dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 35)); + dbfFields.add(new DBFField("reminder", DBFDataType.CHARACTER, 22)); + dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 10)); + dbfFields.add(new DBFField("acc_code", DBFDataType.CHARACTER, 5)); + dbfFields.add(new DBFField("dat", DBFDataType.CHARACTER, 8)); + dbfFields.add(new DBFField("market", DBFDataType.CHARACTER, 1)); + dbfFields.add(new DBFField("acc_name", DBFDataType.CHARACTER, 30)); + dbfFields.add(new DBFField("acc_type", DBFDataType.CHARACTER, 2)); + dbfFields.add(new DBFField("sumengage", DBFDataType.CHARACTER, 22)); + dbfFields.add(new DBFField("sumunblock", DBFDataType.CHARACTER, 22)); + dbfFields.add(new DBFField("file_type", DBFDataType.CHARACTER, 22)); + dbfFields.add(new DBFField("result", DBFDataType.CHARACTER, 3)); + dbfFields.add(new DBFField("gen_time", DBFDataType.DATE)); + dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC)); + dbfFields.add(new DBFField("insdf01_id", DBFDataType.NUMERIC)); + return dbfFields.toArray(DBFField[]::new); + } + +} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF08_Converter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF08_Converter.java new file mode 100644 index 000000000..5c6845cb8 --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF08_Converter.java @@ -0,0 +1,33 @@ +package ru.spcex.clearing.dbf.exporter.services.converters; + +import com.linuxense.javadbf.DBFDataType; +import com.linuxense.javadbf.DBFField; +import org.springframework.stereotype.Service; +import ru.clearing.classes.statics.data.sdf.SDf08; + +import java.util.LinkedList; +import java.util.List; + +@Service +public class S_DF08_Converter extends DFConverter { + @Override + public Object[] toObjectArray(SDf08 entity) { + List values = new LinkedList<>(); + values.add(typeMatch(entity.getNumber())); + values.add(typeMatch(entity.getDatetime())); + values.add(typeMatch(entity.getGenerationTime())); + values.add(typeMatch(entity.getGenerationId())); + return values.toArray(Object[]::new); + } + + @Override + public DBFField[] getDBFHeaders() { + List dbfFields = new LinkedList<>(); + dbfFields.add(new DBFField("number", DBFDataType.NUMERIC, 32)); + dbfFields.add(new DBFField("datetime", DBFDataType.DATE)); + dbfFields.add(new DBFField("gen_time", DBFDataType.DATE)); + dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC)); + return dbfFields.toArray(DBFField[]::new); + } + +} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF12_Converter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF12_Converter.java new file mode 100644 index 000000000..53926b58e --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF12_Converter.java @@ -0,0 +1,37 @@ +package ru.spcex.clearing.dbf.exporter.services.converters; + +import com.linuxense.javadbf.DBFDataType; +import com.linuxense.javadbf.DBFField; +import org.springframework.stereotype.Service; +import ru.clearing.classes.statics.data.sdf.SDf12; + +import java.util.LinkedList; +import java.util.List; + +@Service +public class S_DF12_Converter extends DFConverter { + @Override + public Object[] toObjectArray(SDf12 entity) { + List values = new LinkedList<>(); + values.add(typeMatch(entity.getAccount())); + values.add(typeMatch(entity.getDeal())); + values.add(typeMatch(entity.getStatus())); + values.add(typeMatch(entity.getFileName())); + values.add(typeMatch(entity.getGenerationTime())); + values.add(typeMatch(entity.getGenerationId())); + return values.toArray(Object[]::new); + } + + @Override + public DBFField[] getDBFHeaders() { + List dbfFields = new LinkedList<>(); + dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 25)); + dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 25)); + dbfFields.add(new DBFField("status", DBFDataType.NUMERIC, 32)); + dbfFields.add(new DBFField("file_name", DBFDataType.CHARACTER, 254)); + dbfFields.add(new DBFField("gen_time", DBFDataType.DATE)); + dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC)); + return dbfFields.toArray(DBFField[]::new); + } + +} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF16_Converter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF16_Converter.java new file mode 100644 index 000000000..9bbba8576 --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF16_Converter.java @@ -0,0 +1,43 @@ +package ru.spcex.clearing.dbf.exporter.services.converters; + +import com.linuxense.javadbf.DBFDataType; +import com.linuxense.javadbf.DBFField; +import org.springframework.stereotype.Service; +import ru.clearing.classes.statics.data.sdf.SDf16; + +import java.util.LinkedList; +import java.util.List; + +@Service +public class S_DF16_Converter extends DFConverter { + @Override + public Object[] toObjectArray(SDf16 entity) { + List values = new LinkedList<>(); + values.add(typeMatch(entity.getDate())); + values.add(typeMatch(entity.getAccount())); + values.add(typeMatch(entity.getSum())); + values.add(typeMatch(entity.getMarket())); + values.add(typeMatch(entity.getType())); + values.add(typeMatch(entity.getNumber())); + values.add(typeMatch(entity.getFileName())); + values.add(typeMatch(entity.getGenerationTime())); + values.add(typeMatch(entity.getGenerationId())); + return values.toArray(Object[]::new); + } + + @Override + public DBFField[] getDBFHeaders() { + List dbfFields = new LinkedList<>(); + dbfFields.add(new DBFField("date", DBFDataType.DATE)); + dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 20)); + dbfFields.add(new DBFField("sum", DBFDataType.NUMERIC, 32)); + dbfFields.add(new DBFField("market", DBFDataType.CHARACTER, 1)); + dbfFields.add(new DBFField("type", DBFDataType.CHARACTER, 1)); + dbfFields.add(new DBFField("number", DBFDataType.NUMERIC, 32)); + dbfFields.add(new DBFField("file_name", DBFDataType.CHARACTER, 254)); + dbfFields.add(new DBFField("gen_time", DBFDataType.DATE)); + dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC)); + return dbfFields.toArray(DBFField[]::new); + } + +} diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF18_Converter.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF18_Converter.java new file mode 100644 index 000000000..19193a85c --- /dev/null +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/converters/S_DF18_Converter.java @@ -0,0 +1,39 @@ +package ru.spcex.clearing.dbf.exporter.services.converters; + +import com.linuxense.javadbf.DBFDataType; +import com.linuxense.javadbf.DBFField; +import org.springframework.stereotype.Service; +import ru.clearing.classes.statics.data.sdf.SDf18; + +import java.util.LinkedList; +import java.util.List; + +@Service +public class S_DF18_Converter extends DFConverter { + @Override + public Object[] toObjectArray(SDf18 entity) { + List values = new LinkedList<>(); + values.add(typeMatch(entity.getAccount())); + values.add(typeMatch(entity.getDeal())); + values.add(typeMatch(entity.getStatus())); + values.add(typeMatch(entity.getResult())); + values.add(typeMatch(entity.getGenerationTime())); + values.add(typeMatch(entity.getGenerationId())); + values.add(typeMatch(entity.getInSDf12Id())); + return values.toArray(Object[]::new); + } + + @Override + public DBFField[] getDBFHeaders() { + List dbfFields = new LinkedList<>(); + dbfFields.add(new DBFField("account", DBFDataType.CHARACTER, 25)); + dbfFields.add(new DBFField("deal", DBFDataType.CHARACTER, 4)); + dbfFields.add(new DBFField("status", DBFDataType.NUMERIC, 32)); + dbfFields.add(new DBFField("result", DBFDataType.NUMERIC, 32)); + dbfFields.add(new DBFField("gen_time", DBFDataType.DATE)); + dbfFields.add(new DBFField("gen_id", DBFDataType.NUMERIC)); + dbfFields.add(new DBFField("insdf12_id", DBFDataType.NUMERIC)); + return dbfFields.toArray(DBFField[]::new); + } + +} diff --git a/clearing-parent/dbf-exporter/src/main/resources/application.properties b/clearing-parent/dbf-exporter/src/main/resources/application.properties index 089043691..5fc2e1ad8 100644 --- a/clearing-parent/dbf-exporter/src/main/resources/application.properties +++ b/clearing-parent/dbf-exporter/src/main/resources/application.properties @@ -2,10 +2,9 @@ server.port=8080 server.servlet.context-path=/exporter spring.main.web-application-type=servlet -db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres -db.driver=org.postgresql.Driver -db.login=clearing -db.password=Aa111111 +hazelcast.cluster-members=127.0.0.1:5701 +hazelcast.login=dev +hazelcast.password=dev-pass dbf.encoding=cp866 dbf.threads-count=10