From af1d4db881ea40270418550b6818c517dcfcf194 Mon Sep 17 00:00:00 2001 From: akulikov Date: Mon, 1 Aug 2022 16:18:14 +0300 Subject: [PATCH] add dbf export module --- .../dbf/export/config/DBFExportConfig.java | 4 +- .../export/logic/data/ResultContainer.java | 1 + .../export/logic/stages/CreateDBFFiles.java | 53 -------- .../dbf/export/logic/stages/ExportFromDB.java | 124 +++++++++++++++++- .../export/logic/stages/PrepareDBFFiles.java | 69 ++++++++++ .../dbf/export/properties/AProperties.java | 58 +------- .../dbf/export/services/DBFExportService.java | 2 +- .../src/main/resources/dbf_export.properties | 6 +- 8 files changed, 204 insertions(+), 113 deletions(-) delete mode 100644 clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java create mode 100644 clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/PrepareDBFFiles.java diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java index 44b4d3c5d..d82e773fd 100644 --- a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java @@ -8,6 +8,7 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.jdbc.core.JdbcTemplate; import ru.spcex.clearing.dbf.export.logic.stages.ExportFromDB; +import ru.spcex.clearing.dbf.export.logic.stages.PrepareDBFFiles; import ru.spcex.clearing.dbf.export.logic.stages.Stage; import ru.spcex.clearing.dbf.export.properties.AProperties; @@ -22,7 +23,7 @@ public class DBFExportConfig { private final AProperties properties; private final ApplicationContext context; - public DBFExportConfig(@Qualifier("dbfLoaderProperties") AProperties properties, ApplicationContext context) { + public DBFExportConfig(@Qualifier("dbfExportProperties") AProperties properties, ApplicationContext context) { this.properties = properties; this.context = context; } @@ -46,6 +47,7 @@ public class DBFExportConfig { public List pipeline() { List pipeline = new LinkedList<>(); + pipeline.add(context.getBean(PrepareDBFFiles.class)); pipeline.add(context.getBean(ExportFromDB.class)); return pipeline; diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java index 85bc97d82..bc8adc5da 100644 --- a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java @@ -31,4 +31,5 @@ public class ResultContainer { public void setFilesForTables(Map filesForTables) { this.filesForTables = filesForTables; } + } diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java deleted file mode 100644 index 0d4ca0194..000000000 --- a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java +++ /dev/null @@ -1,53 +0,0 @@ -package ru.spcex.clearing.dbf.export.logic.stages; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.stereotype.Component; -import ru.spcex.clearing.dbf.export.exceptions.ConfigException; -import ru.spcex.clearing.dbf.export.logic.data.ResultContainer; -import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult; -import ru.spcex.clearing.dbf.export.logic.data.enums.Table; -import ru.spcex.clearing.dbf.export.properties.AProperties; - -import java.io.File; -import java.nio.charset.Charset; -import java.time.LocalDateTime; -import java.util.HashMap; -import java.util.Map; - -/** - * Создание DBF файлов - */ -@Component -public class CreateDBFFiles extends Stage implements InitializingBean { - private final AProperties properties; - - public CreateDBFFiles(AProperties properties) { - this.properties = properties; - } - - @Override - public StageResult process(ResultContainer resultContainer) { - Map dbfFiles = new HashMap<>(); - for (Table table : resultContainer.getTablesForExport().keySet()) { - File dbfFile = new File(table.getFilePrefix() + "_" + LocalDateTime.now()); - } - return StageResult.OK; - } - - /** - * 1. Собирает из БД названия столбцов для дальнейшей валидации - * 2. Проверяет кодировку из настройки dbf.encoding - */ - @Override - public void afterPropertiesSet() throws Exception { - initDBFCharset(); - } - - private void initDBFCharset() { - try { - Charset.forName(properties.getDbfEncoding()); - } catch (Exception e) { - throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e); - } - } -} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java index 97a36190f..1922c49f2 100644 --- a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java @@ -1,21 +1,44 @@ package ru.spcex.clearing.dbf.export.logic.stages; +import com.linuxense.javadbf.DBFDataType; +import com.linuxense.javadbf.DBFException; +import com.linuxense.javadbf.DBFField; +import com.linuxense.javadbf.DBFWriter; +import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.JdbcUtils; +import org.springframework.jdbc.support.MetaDataAccessException; import org.springframework.stereotype.Component; +import ru.spcex.clearing.dbf.export.exceptions.ConfigException; +import ru.spcex.clearing.dbf.export.logic.data.ISqlFilter; import ru.spcex.clearing.dbf.export.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.export.logic.data.enums.ColumnType; import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.export.logic.data.enums.Table; import ru.spcex.clearing.dbf.export.properties.AProperties; +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.sql.ResultSet; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.*; + /** * Выгрузка данных из базы и их запись */ @Component -public class ExportFromDB extends Stage { +public class ExportFromDB extends Stage implements InitializingBean { private final AProperties properties; private final JdbcTemplate dbfJdbcTemplate; + private final Map dbfFieldsForTable = new HashMap<>(); - public ExportFromDB(@Qualifier("dbfLoaderProperties") AProperties properties, + private Charset dbfCharset; + + public ExportFromDB(@Qualifier("dbfExportProperties") AProperties properties, @Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) { this.properties = properties; this.dbfJdbcTemplate = dbfJdbcTemplate; @@ -23,7 +46,104 @@ public class ExportFromDB extends Stage { @Override public StageResult process(ResultContainer resultContainer) { + for (Table table : resultContainer.getTablesForExport().keySet()) { + ISqlFilter sqlFilter = resultContainer.getTablesForExport().get(table); + String sqlCondition = sqlFilter != null ? " where " + sqlFilter : ""; + + String sql = "select * from " + table + sqlCondition; + List> recordsFromDB = dbfJdbcTemplate.queryForList(sql); + + File dbfFile = resultContainer.getFilesForTables().get(table); + DBFField[] dbfFields = dbfFieldsForTable.get(table); + + try { + boolean createOk = dbfFile.createNewFile(); + if (!createOk) throw new IOException("Can't create " + dbfFile.getName() + " file"); + } catch (IOException e) { + log.error("Can't create file " + dbfFile + " for table " + table + ". Table was skipped.", e); + continue; + } + + try (DBFWriter dbfWriter = new DBFWriter(dbfFile, dbfCharset)) { + dbfWriter.setFields(dbfFields); + for (Map record : recordsFromDB) { + int columnCount = dbfFields.length; + Object[] values = new Object[columnCount]; + for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) { + DBFDataType currDBFType = dbfFields[columnIdx].getType(); + Object currColumn = record.get(dbfFields[columnIdx].getName()); + + if (currDBFType == DBFDataType.LOGICAL) { + values[columnIdx] = Boolean.parseBoolean(String.valueOf(currColumn)); + + } else if (currDBFType == DBFDataType.DATE) { + LocalDate date = LocalDate.parse(String.valueOf(currColumn)); + values[columnIdx] = new Date(date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli()); + + } else if (currDBFType == DBFDataType.NUMERIC || currDBFType == DBFDataType.FLOATING_POINT) { + BigDecimal bigDecimal = new BigDecimal(String.valueOf(currColumn)); + values[columnIdx] = bigDecimal; + + } else { + values[columnIdx] = String.valueOf(currColumn); + + } + } + dbfWriter.addRecord(values); + } + } catch (DBFException e) { + log.error("Can't export table " + table + " to file " + dbfFile + ". Table was skipped.", e); + } + } return StageResult.OK; } + + + /** + * 1. Собирает структуру БД для дальнейшей записи DBF файлов + * 2. Проверяет кодировку из настройки dbf.encoding + */ + @Override + public void afterPropertiesSet() throws Exception { + initDBStructure(); + initDBFCharset(); + } + + private void initDBStructure() throws MetaDataAccessException { + boolean ok = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()), + dbMeta -> { + List tableNamesFromResultSet = new LinkedList<>(); + ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"}); + while (tableNamesRS.next()) { + tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME")); + } + for (String tableNameFromResultSet : tableNamesFromResultSet) { + Table currTable = Table.tableForName(tableNameFromResultSet); + if (currTable == null) continue; + List dbfFields = new ArrayList<>(); + ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null); + while (columnNamesRS.next()) { + String name = columnNamesRS.getString("COLUMN_NAME"); + ColumnType type = ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE")); + if (type == null) throw new ConfigException("Unsupported ColumnType from DB"); + int length = columnNamesRS.getInt("COLUMN_SIZE"); + if (type == ColumnType.DATE) length = 8; + int decimalCount = columnNamesRS.getInt("DECIMAL_DIGITS"); + DBFField dbfField = new DBFField(name.toUpperCase(), type.getDbfType(), length, decimalCount); + dbfFields.add(dbfField); + } + dbfFieldsForTable.put(currTable, dbfFields.toArray(DBFField[]::new)); + } + return true; + }); + } + + private void initDBFCharset() { + try { + dbfCharset = Charset.forName(properties.getDbfEncoding()); + } catch (Exception e) { + throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e); + } + } } diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/PrepareDBFFiles.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/PrepareDBFFiles.java new file mode 100644 index 000000000..3215f02d9 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/PrepareDBFFiles.java @@ -0,0 +1,69 @@ +package ru.spcex.clearing.dbf.export.logic.stages; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import ru.spcex.clearing.dbf.export.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.export.logic.data.enums.Table; +import ru.spcex.clearing.dbf.export.properties.AProperties; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; + +/** + * Создание DBF файлов + */ +@Component +public class PrepareDBFFiles extends Stage implements InitializingBean { + private final AProperties properties; + private String outDirWithDateTime; + + public PrepareDBFFiles(@Qualifier("dbfExportProperties") AProperties properties) { + this.properties = properties; + } + + @Override + public StageResult process(ResultContainer resultContainer) { + Map dbfFiles = new HashMap<>(); + for (Table table : resultContainer.getTablesForExport().keySet()) { + File dbfFile = new File(String.format("%s%s%s.dbf", + outDirWithDateTime, + File.separator, + table.getFilePrefix())); + try { + if (dbfFile.exists()) { + boolean deleteOk = dbfFile.delete(); + if (!deleteOk) throw new IOException("Can't overwrite " + dbfFile.getName() + " file"); + } + + dbfFiles.put(table, dbfFile); + } catch (IOException e) { + log.error("Can't create " + dbfFile + ". Table was skipped", e); + resultContainer.getTablesForExport().remove(table); + } + } + resultContainer.setFilesForTables(dbfFiles); + return StageResult.OK; + } + + + @Override + public void afterPropertiesSet() throws Exception { + String outDirPath = properties.getOutDir(); + File outDir = new File(String.format("%s%s%s", + outDirPath, + File.separator, + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmmss")))); + if (outDir.exists()) throw new IOException("Output dir " + outDir + " already exist."); + + boolean createDirOk = outDir.mkdirs(); + if (!createDirOk) throw new IOException("Can't create output dir " + outDir); + + outDirWithDateTime = outDir.getAbsolutePath(); + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java index 497b90c39..d5508668a 100644 --- a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java @@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; -@Component("dbfLoaderProperties") +@Component("dbfExportProperties") @PropertySource(value = {"classpath:dbf_export.properties"}) @PropertySource(value = {"file:dbf_export.properties"}, ignoreResourceNotFound = true) public class AProperties { @@ -21,24 +21,12 @@ public class AProperties { @Value("${db.password}") private String dbPassword; - @Value("${dbf.src-dir}") - private String srcDir; - - @Value("${dbf.delete-src-files}") - private boolean deleteSrcFiles = true; - - @Value("${dbf.check-source-timeout}") - private long checkSourceTimeout; - - @Value("${dbf.execute-timeout}") - private long executeTimeout; + @Value("${dbf.out-dir}") + private String outDir; @Value("${dbf.encoding}") private String dbfEncoding; - @Value("${dbf.insert-batch-size}") - private int insertBatchSize; - public String getJdbcUrl() { return jdbcUrl; } @@ -71,28 +59,12 @@ public class AProperties { this.dbDriver = dbDriver; } - public String getSrcDir() { - return srcDir; + public String getOutDir() { + return outDir; } - public void setSrcDir(String srcDir) { - this.srcDir = srcDir; - } - - public long getCheckSourceTimeout() { - return checkSourceTimeout; - } - - public void setCheckSourceTimeout(long checkSourceTimeout) { - this.checkSourceTimeout = checkSourceTimeout; - } - - public boolean deleteSrcFiles() { - return deleteSrcFiles; - } - - public void setDeleteSrcFiles(boolean deleteSrcFiles) { - this.deleteSrcFiles = deleteSrcFiles; + public void setOutDir(String outDir) { + this.outDir = outDir; } public String getDbfEncoding() { @@ -102,20 +74,4 @@ public class AProperties { public void setDbfEncoding(String dbfEncoding) { this.dbfEncoding = dbfEncoding; } - - public long getExecuteTimeout() { - return executeTimeout; - } - - public void setExecuteTimeout(long executeTimeout) { - this.executeTimeout = executeTimeout; - } - - public int getInsertBatchSize() { - return insertBatchSize; - } - - public void setInsertBatchSize(int insertBatchSize) { - this.insertBatchSize = insertBatchSize; - } } diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java index d6713d489..c55d2936d 100644 --- a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java @@ -31,7 +31,7 @@ public class DBFExportService { @Scheduled(fixedDelay = 9999999999999999L) public void run() { Map tablesForExport = new HashMap<>(); - for (Table table : Table.values()) tablesForExport.put(table, () -> ""); + for (Table table : Table.values()) tablesForExport.put(table, null); ResultContainer resultContainer = ResultContainer.createNewTask(tablesForExport); for (Stage stage : pipeline) { StageResult result = stage.process(resultContainer); diff --git a/clearing-parent/dbf-export/src/main/resources/dbf_export.properties b/clearing-parent/dbf-export/src/main/resources/dbf_export.properties index 0e309b572..de1e33507 100644 --- a/clearing-parent/dbf-export/src/main/resources/dbf_export.properties +++ b/clearing-parent/dbf-export/src/main/resources/dbf_export.properties @@ -3,10 +3,6 @@ db.driver=org.postgresql.Driver db.login=clearing db.password=Aa111111 -dbf.check-source-timeout=300000000 -dbf.execute-timeout=300000000 dbf.encoding=cp866 -dbf.insert-batch-size=100 -dbf.delete-src-files=false -dbf.src-dir=D:\\dbf\\ +dbf.out-dir=D:\\dbf\\out