treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add dbf export module

Browse source
This commit is contained in:
akulikov 2022-08-01 16:18:14 +03:00
parent 36d1cc3d99
commit af1d4db881
8 changed files with 204 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java
View file

@ -8,6 +8,7 @@ import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;
import ru.spcex.clearing.dbf.export.logic.stages.ExportFromDB;
import ru.spcex.clearing.dbf.export.logic.stages.PrepareDBFFiles;
import ru.spcex.clearing.dbf.export.logic.stages.Stage;
import ru.spcex.clearing.dbf.export.properties.AProperties;
@ -22,7 +23,7 @@ public class DBFExportConfig {
private final AProperties properties;
private final ApplicationContext context;
public DBFExportConfig(@Qualifier("dbfLoaderProperties") AProperties properties, ApplicationContext context) {
public DBFExportConfig(@Qualifier("dbfExportProperties") AProperties properties, ApplicationContext context) {
this.properties = properties;
this.context = context;
}
@ -46,6 +47,7 @@ public class DBFExportConfig {
public List<Stage> pipeline() {
List<Stage> pipeline = new LinkedList<>();
pipeline.add(context.getBean(PrepareDBFFiles.class));
pipeline.add(context.getBean(ExportFromDB.class));
return pipeline;

1
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java
View file

@ -31,4 +31,5 @@ public class ResultContainer {
public void setFilesForTables(Map<Table, File> filesForTables) {
this.filesForTables = filesForTables;
}
}

53
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java
View file

@ -1,53 +0,0 @@
package ru.spcex.clearing.dbf.export.logic.stages;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.export.exceptions.ConfigException;
import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
import ru.spcex.clearing.dbf.export.properties.AProperties;
import java.io.File;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
/**
* Создание DBF файлов
*/
@Component
public class CreateDBFFiles extends Stage implements InitializingBean {
private final AProperties properties;
public CreateDBFFiles(AProperties properties) {
this.properties = properties;
}
@Override
public StageResult process(ResultContainer resultContainer) {
Map<Table, File> dbfFiles = new HashMap<>();
for (Table table : resultContainer.getTablesForExport().keySet()) {
File dbfFile = new File(table.getFilePrefix() + "_" + LocalDateTime.now());
}
return StageResult.OK;
}
/**
* 1. Собирает из БД названия столбцов для дальнейшей валидации
* 2. Проверяет кодировку из настройки dbf.encoding
*/
@Override
public void afterPropertiesSet() throws Exception {
initDBFCharset();
}
private void initDBFCharset() {
try {
Charset.forName(properties.getDbfEncoding());
} catch (Exception e) {
throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
}
}
}

124
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java
View file

@ -1,21 +1,44 @@
package ru.spcex.clearing.dbf.export.logic.stages;
import com.linuxense.javadbf.DBFDataType;
import com.linuxense.javadbf.DBFException;
import com.linuxense.javadbf.DBFField;
import com.linuxense.javadbf.DBFWriter;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.JdbcUtils;
import org.springframework.jdbc.support.MetaDataAccessException;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.export.exceptions.ConfigException;
import ru.spcex.clearing.dbf.export.logic.data.ISqlFilter;
import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.export.logic.data.enums.ColumnType;
import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
import ru.spcex.clearing.dbf.export.properties.AProperties;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.sql.ResultSet;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.*;
/**
* Выгрузка данных из базы и их запись
*/
@Component
public class ExportFromDB extends Stage {
public class ExportFromDB extends Stage implements InitializingBean {
private final AProperties properties;
private final JdbcTemplate dbfJdbcTemplate;
private final Map<Table, DBFField[]> dbfFieldsForTable = new HashMap<>();
public ExportFromDB(@Qualifier("dbfLoaderProperties") AProperties properties,
private Charset dbfCharset;
public ExportFromDB(@Qualifier("dbfExportProperties") AProperties properties,
@Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) {
this.properties = properties;
this.dbfJdbcTemplate = dbfJdbcTemplate;
@ -23,7 +46,104 @@ public class ExportFromDB extends Stage {
@Override
public StageResult process(ResultContainer resultContainer) {
for (Table table : resultContainer.getTablesForExport().keySet()) {
ISqlFilter sqlFilter = resultContainer.getTablesForExport().get(table);
String sqlCondition = sqlFilter != null ? " where " + sqlFilter : "";
String sql = "select * from " + table + sqlCondition;
List<Map<String, Object>> recordsFromDB = dbfJdbcTemplate.queryForList(sql);
File dbfFile = resultContainer.getFilesForTables().get(table);
DBFField[] dbfFields = dbfFieldsForTable.get(table);
try {
boolean createOk = dbfFile.createNewFile();
if (!createOk) throw new IOException("Can't create " + dbfFile.getName() + " file");
} catch (IOException e) {
log.error("Can't create file " + dbfFile + " for table " + table + ". Table was skipped.", e);
continue;
}
try (DBFWriter dbfWriter = new DBFWriter(dbfFile, dbfCharset)) {
dbfWriter.setFields(dbfFields);
for (Map<String, Object> record : recordsFromDB) {
int columnCount = dbfFields.length;
Object[] values = new Object[columnCount];
for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) {
DBFDataType currDBFType = dbfFields[columnIdx].getType();
Object currColumn = record.get(dbfFields[columnIdx].getName());
if (currDBFType == DBFDataType.LOGICAL) {
values[columnIdx] = Boolean.parseBoolean(String.valueOf(currColumn));
} else if (currDBFType == DBFDataType.DATE) {
LocalDate date = LocalDate.parse(String.valueOf(currColumn));
values[columnIdx] = new Date(date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli());
} else if (currDBFType == DBFDataType.NUMERIC || currDBFType == DBFDataType.FLOATING_POINT) {
BigDecimal bigDecimal = new BigDecimal(String.valueOf(currColumn));
values[columnIdx] = bigDecimal;
} else {
values[columnIdx] = String.valueOf(currColumn);
}
}
dbfWriter.addRecord(values);
}
} catch (DBFException e) {
log.error("Can't export table " + table + " to file " + dbfFile + ". Table was skipped.", e);
}
}
return StageResult.OK;
}
/**
* 1. Собирает структуру БД для дальнейшей записи DBF файлов
* 2. Проверяет кодировку из настройки dbf.encoding
*/
@Override
public void afterPropertiesSet() throws Exception {
initDBStructure();
initDBFCharset();
}
private void initDBStructure() throws MetaDataAccessException {
boolean ok = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()),
dbMeta -> {
List<String> tableNamesFromResultSet = new LinkedList<>();
ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"});
while (tableNamesRS.next()) {
tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME"));
}
for (String tableNameFromResultSet : tableNamesFromResultSet) {
Table currTable = Table.tableForName(tableNameFromResultSet);
if (currTable == null) continue;
List<DBFField> dbfFields = new ArrayList<>();
ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null);
while (columnNamesRS.next()) {
String name = columnNamesRS.getString("COLUMN_NAME");
ColumnType type = ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE"));
if (type == null) throw new ConfigException("Unsupported ColumnType from DB");
int length = columnNamesRS.getInt("COLUMN_SIZE");
if (type == ColumnType.DATE) length = 8;
int decimalCount = columnNamesRS.getInt("DECIMAL_DIGITS");
DBFField dbfField = new DBFField(name.toUpperCase(), type.getDbfType(), length, decimalCount);
dbfFields.add(dbfField);
}
dbfFieldsForTable.put(currTable, dbfFields.toArray(DBFField[]::new));
}
return true;
});
}
private void initDBFCharset() {
try {
dbfCharset = Charset.forName(properties.getDbfEncoding());
} catch (Exception e) {
throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
}
}
}

69
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/PrepareDBFFiles.java Normal file
View file

@ -0,0 +1,69 @@
package ru.spcex.clearing.dbf.export.logic.stages;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
import ru.spcex.clearing.dbf.export.properties.AProperties;
import java.io.File;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Map;
/**
* Создание DBF файлов
*/
@Component
public class PrepareDBFFiles extends Stage implements InitializingBean {
private final AProperties properties;
private String outDirWithDateTime;
public PrepareDBFFiles(@Qualifier("dbfExportProperties") AProperties properties) {
this.properties = properties;
}
@Override
public StageResult process(ResultContainer resultContainer) {
Map<Table, File> dbfFiles = new HashMap<>();
for (Table table : resultContainer.getTablesForExport().keySet()) {
File dbfFile = new File(String.format("%s%s%s.dbf",
outDirWithDateTime,
File.separator,
table.getFilePrefix()));
try {
if (dbfFile.exists()) {
boolean deleteOk = dbfFile.delete();
if (!deleteOk) throw new IOException("Can't overwrite " + dbfFile.getName() + " file");
}
dbfFiles.put(table, dbfFile);
} catch (IOException e) {
log.error("Can't create " + dbfFile + ". Table was skipped", e);
resultContainer.getTablesForExport().remove(table);
}
}
resultContainer.setFilesForTables(dbfFiles);
return StageResult.OK;
}
@Override
public void afterPropertiesSet() throws Exception {
String outDirPath = properties.getOutDir();
File outDir = new File(String.format("%s%s%s",
outDirPath,
File.separator,
LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmmss"))));
if (outDir.exists()) throw new IOException("Output dir " + outDir + " already exist.");
boolean createDirOk = outDir.mkdirs();
if (!createDirOk) throw new IOException("Can't create output dir " + outDir);
outDirWithDateTime = outDir.getAbsolutePath();
}
}

58
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java
View file

@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
@Component("dbfLoaderProperties")
@Component("dbfExportProperties")
@PropertySource(value = {"classpath:dbf_export.properties"})
@PropertySource(value = {"file:dbf_export.properties"}, ignoreResourceNotFound = true)
public class AProperties {
@ -21,24 +21,12 @@ public class AProperties {
@Value("${db.password}")
private String dbPassword;
@Value("${dbf.src-dir}")
private String srcDir;
@Value("${dbf.delete-src-files}")
private boolean deleteSrcFiles = true;
@Value("${dbf.check-source-timeout}")
private long checkSourceTimeout;
@Value("${dbf.execute-timeout}")
private long executeTimeout;
@Value("${dbf.out-dir}")
private String outDir;
@Value("${dbf.encoding}")
private String dbfEncoding;
@Value("${dbf.insert-batch-size}")
private int insertBatchSize;
public String getJdbcUrl() {
return jdbcUrl;
}
@ -71,28 +59,12 @@ public class AProperties {
this.dbDriver = dbDriver;
}
public String getSrcDir() {
return srcDir;
public String getOutDir() {
return outDir;
}
public void setSrcDir(String srcDir) {
this.srcDir = srcDir;
}
public long getCheckSourceTimeout() {
return checkSourceTimeout;
}
public void setCheckSourceTimeout(long checkSourceTimeout) {
this.checkSourceTimeout = checkSourceTimeout;
}
public boolean deleteSrcFiles() {
return deleteSrcFiles;
}
public void setDeleteSrcFiles(boolean deleteSrcFiles) {
this.deleteSrcFiles = deleteSrcFiles;
public void setOutDir(String outDir) {
this.outDir = outDir;
}
public String getDbfEncoding() {
@ -102,20 +74,4 @@ public class AProperties {
public void setDbfEncoding(String dbfEncoding) {
this.dbfEncoding = dbfEncoding;
}
public long getExecuteTimeout() {
return executeTimeout;
}
public void setExecuteTimeout(long executeTimeout) {
this.executeTimeout = executeTimeout;
}
public int getInsertBatchSize() {
return insertBatchSize;
}
public void setInsertBatchSize(int insertBatchSize) {
this.insertBatchSize = insertBatchSize;
}
}

2
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java
View file

@ -31,7 +31,7 @@ public class DBFExportService {
@Scheduled(fixedDelay = 9999999999999999L)
public void run() {
Map<Table, ISqlFilter> tablesForExport = new HashMap<>();
for (Table table : Table.values()) tablesForExport.put(table, () -> "");
for (Table table : Table.values()) tablesForExport.put(table, null);
ResultContainer resultContainer = ResultContainer.createNewTask(tablesForExport);
for (Stage stage : pipeline) {
StageResult result = stage.process(resultContainer);

6
clearing-parent/dbf-export/src/main/resources/dbf_export.properties
View file

@ -3,10 +3,6 @@ db.driver=org.postgresql.Driver
db.login=clearing
db.password=Aa111111
dbf.check-source-timeout=300000000
dbf.execute-timeout=300000000
dbf.encoding=cp866
dbf.insert-batch-size=100
dbf.delete-src-files=false
dbf.src-dir=D:\\dbf\\
dbf.out-dir=D:\\dbf\\out