treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add dbf export module (in progress)

Browse source
This commit is contained in:
akulikov 2022-08-01 13:56:47 +03:00
parent 04087dd313
commit 36d1cc3d99
17 changed files with 605 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
clearing-parent/dbf-export/pom.xml Normal file
View file

@ -0,0 +1,86 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>dbf-export</artifactId>
<name>dbf-export</name>
<description>DBF exporter for DBF files</description>
<version>SPCEX-1.0.0.0</version>
<parent>
<artifactId>clearing-parent</artifactId>
<groupId>ru.spcex.clearing</groupId>
<version>SPCEX-1.0.0.0</version>
</parent>
<dependencies>
<!-- Spring boot -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<!-- JDBC -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</dependency>
<dependency>
<groupId>com.mchange</groupId>
<artifactId>c3p0</artifactId>
<version>0.9.5.2</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<!-- DBF files -->
<dependency>
<groupId>com.github.albfernandez</groupId>
<artifactId>javadbf</artifactId>
<version>1.13.1</version>
</dependency>
</dependencies>
<build>
<finalName>jar/${project.artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<!--<classpathPrefix>../lib/</classpathPrefix>-->
<mainClass>ru.spcex.clearing.dbf.export.DBFLoaderApplication</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
<configuration>
<addResources>true</addResources>
<classifier>exec</classifier>
</configuration>
</plugin>
</plugins>
</build>
</project>

18
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/DBFExportApplication.java Normal file
View file

@ -0,0 +1,18 @@
package ru.spcex.clearing.dbf.export;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
@SpringBootApplication
public class DBFExportApplication {
public static void main(String[] args) {
try {
SpringApplicationBuilder builder = new SpringApplicationBuilder(DBFExportApplication.class);
builder.run(args);
} catch (Throwable e) {
LoggerFactory.getLogger(DBFExportApplication.class).error("DBF-Loader start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage());
System.exit(-1);
}
}
}

54
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java Normal file
View file

@ -0,0 +1,54 @@
package ru.spcex.clearing.dbf.export.config;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;
import ru.spcex.clearing.dbf.export.logic.stages.ExportFromDB;
import ru.spcex.clearing.dbf.export.logic.stages.Stage;
import ru.spcex.clearing.dbf.export.properties.AProperties;
import javax.sql.DataSource;
import java.beans.PropertyVetoException;
import java.util.LinkedList;
import java.util.List;
@Configuration
@ComponentScan(basePackages = {"ru.spcex.clearing.dbf.export"})
public class DBFExportConfig {
private final AProperties properties;
private final ApplicationContext context;
public DBFExportConfig(@Qualifier("dbfLoaderProperties") AProperties properties, ApplicationContext context) {
this.properties = properties;
this.context = context;
}
@Bean("dbfDataSource")
public DataSource dataSource() throws PropertyVetoException {
ComboPooledDataSource dataSource = new ComboPooledDataSource();
dataSource.setDriverClass(properties.getDbDriver());
dataSource.setJdbcUrl(properties.getJdbcUrl());
dataSource.setUser(properties.getDbLogin());
dataSource.setPassword(properties.getDbPassword());
return dataSource;
}
@Bean("dbfJdbcTemplate")
public JdbcTemplate jdbcTemplate(@Qualifier("dbfDataSource") DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
@Bean("pipeline")
public List<Stage> pipeline() {
List<Stage> pipeline = new LinkedList<>();
pipeline.add(context.getBean(ExportFromDB.class));
return pipeline;
}
}

6
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/exceptions/ConfigException.java Normal file
View file

@ -0,0 +1,6 @@
package ru.spcex.clearing.dbf.export.exceptions;
public class ConfigException extends RuntimeException {
public ConfigException(String msg) { super(msg); }
public ConfigException(String msg, Throwable cause) { super(msg, cause); }
}

8
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ISqlFilter.java Normal file
View file

@ -0,0 +1,8 @@
package ru.spcex.clearing.dbf.export.logic.data;
/**
* Фильтр записей для экспорта
*/
public interface ISqlFilter {
String getSqlCondition();
}

34
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java Normal file
View file

@ -0,0 +1,34 @@
package ru.spcex.clearing.dbf.export.logic.data;
import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
import java.io.File;
import java.util.Map;
/**
* Контейнер для передачи результата между стадиями
*/
public class ResultContainer {
private Map<Table, ISqlFilter> tablesForExport;
private Map<Table, File> filesForTables;
protected ResultContainer() {}
public static ResultContainer createNewTask(Map<Table, ISqlFilter> tablesForExport) {
ResultContainer container = new ResultContainer();
container.tablesForExport = tablesForExport;
return container;
}
public Map<Table, ISqlFilter> getTablesForExport() {
return tablesForExport;
}
public Map<Table, File> getFilesForTables() {
return filesForTables;
}
public void setFilesForTables(Map<Table, File> filesForTables) {
this.filesForTables = filesForTables;
}
}

46
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/ColumnType.java Normal file
View file

@ -0,0 +1,46 @@
package ru.spcex.clearing.dbf.export.logic.data.enums;
import com.linuxense.javadbf.DBFDataType;
import java.sql.Types;
/**
* Типы данных в таблицах.
* Ставит в соответствие типы PostGRE и DBF
*/
public enum ColumnType {
VARCHAR(DBFDataType.CHARACTER, Types.VARCHAR),
CHARACTER(DBFDataType.CHARACTER, Types.CHAR),
NUMERIC(DBFDataType.NUMERIC, Types.NUMERIC),
DATE(DBFDataType.DATE, Types.DATE);
private final DBFDataType dbfType;
private final int sqlType;
ColumnType(DBFDataType dbfType, int postgreSqlType) {
this.dbfType = dbfType;
this.sqlType = postgreSqlType;
}
public DBFDataType getDbfType() {
return dbfType;
}
public int getSqlType() {
return sqlType;
}
public static ColumnType getForSQLType(int sqlType) {
for (ColumnType columnType : values()) {
if (columnType.sqlType == sqlType) return columnType;
}
return null;
}
public static ColumnType getForDBFType(DBFDataType dbfType) {
for (ColumnType columnType : values()) {
if (columnType.dbfType == dbfType) return columnType;
}
return null;
}
}

7
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/StageResult.java Normal file
View file

@ -0,0 +1,7 @@
package ru.spcex.clearing.dbf.export.logic.data.enums;
public enum StageResult {
OK,
ERROR,
COMPLETE
}

39
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/Table.java Normal file
View file

@ -0,0 +1,39 @@
package ru.spcex.clearing.dbf.export.logic.data.enums;
public enum Table {
DF_01("DF-01"),
DF_02("DF-02"),
DF_03("DF-03"),
DF_04("DF-04"),
DF_05("DF-05"),
DF_06("DF-06");
private final String filePrefix;
Table(String prefix) {
this.filePrefix = prefix;
}
public boolean fileForThisTable(String filename) {
return filename != null && filename.startsWith(filePrefix);
}
public static Table getTableForFilename(String filename) {
for (Table table : Table.values()) {
if (table.fileForThisTable(filename)) return table;
}
return null;
}
public static Table tableForName(String name) {
for (Table table : values()) {
if (name.equalsIgnoreCase(table.name()))
return table;
}
return null;
}
public String getFilePrefix() {
return filePrefix;
}
}

53
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java Normal file
View file

@ -0,0 +1,53 @@
package ru.spcex.clearing.dbf.export.logic.stages;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.export.exceptions.ConfigException;
import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
import ru.spcex.clearing.dbf.export.properties.AProperties;
import java.io.File;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
/**
* Создание DBF файлов
*/
@Component
public class CreateDBFFiles extends Stage implements InitializingBean {
private final AProperties properties;
public CreateDBFFiles(AProperties properties) {
this.properties = properties;
}
@Override
public StageResult process(ResultContainer resultContainer) {
Map<Table, File> dbfFiles = new HashMap<>();
for (Table table : resultContainer.getTablesForExport().keySet()) {
File dbfFile = new File(table.getFilePrefix() + "_" + LocalDateTime.now());
}
return StageResult.OK;
}
/**
* 1. Собирает из БД названия столбцов для дальнейшей валидации
* 2. Проверяет кодировку из настройки dbf.encoding
*/
@Override
public void afterPropertiesSet() throws Exception {
initDBFCharset();
}
private void initDBFCharset() {
try {
Charset.forName(properties.getDbfEncoding());
} catch (Exception e) {
throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
}
}
}

29
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java Normal file
View file

@ -0,0 +1,29 @@
package ru.spcex.clearing.dbf.export.logic.stages;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.export.properties.AProperties;
/**
* Выгрузка данных из базы и их запись
*/
@Component
public class ExportFromDB extends Stage {
private final AProperties properties;
private final JdbcTemplate dbfJdbcTemplate;
public ExportFromDB(@Qualifier("dbfLoaderProperties") AProperties properties,
@Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) {
this.properties = properties;
this.dbfJdbcTemplate = dbfJdbcTemplate;
}
@Override
public StageResult process(ResultContainer resultContainer) {
return StageResult.OK;
}
}

12
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/Stage.java Normal file
View file

@ -0,0 +1,12 @@
package ru.spcex.clearing.dbf.export.logic.stages;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
public abstract class Stage {
protected Logger log = LoggerFactory.getLogger(getClass());
public abstract StageResult process(ResultContainer resultContainer);
}

121
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java Normal file
View file

@ -0,0 +1,121 @@
package ru.spcex.clearing.dbf.export.properties;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
@Component("dbfLoaderProperties")
@PropertySource(value = {"classpath:dbf_export.properties"})
@PropertySource(value = {"file:dbf_export.properties"}, ignoreResourceNotFound = true)
public class AProperties {
@Value("${db.jdbc-url}")
private String jdbcUrl;
@Value("${db.driver}")
private String dbDriver;
@Value("${db.login}")
private String dbLogin;
@Value("${db.password}")
private String dbPassword;
@Value("${dbf.src-dir}")
private String srcDir;
@Value("${dbf.delete-src-files}")
private boolean deleteSrcFiles = true;
@Value("${dbf.check-source-timeout}")
private long checkSourceTimeout;
@Value("${dbf.execute-timeout}")
private long executeTimeout;
@Value("${dbf.encoding}")
private String dbfEncoding;
@Value("${dbf.insert-batch-size}")
private int insertBatchSize;
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getDbLogin() {
return dbLogin;
}
public void setDbLogin(String dbLogin) {
this.dbLogin = dbLogin;
}
public String getDbPassword() {
return dbPassword;
}
public void setDbPassword(String dbPassword) {
this.dbPassword = dbPassword;
}
public String getDbDriver() {
return dbDriver;
}
public void setDbDriver(String dbDriver) {
this.dbDriver = dbDriver;
}
public String getSrcDir() {
return srcDir;
}
public void setSrcDir(String srcDir) {
this.srcDir = srcDir;
}
public long getCheckSourceTimeout() {
return checkSourceTimeout;
}
public void setCheckSourceTimeout(long checkSourceTimeout) {
this.checkSourceTimeout = checkSourceTimeout;
}
public boolean deleteSrcFiles() {
return deleteSrcFiles;
}
public void setDeleteSrcFiles(boolean deleteSrcFiles) {
this.deleteSrcFiles = deleteSrcFiles;
}
public String getDbfEncoding() {
return dbfEncoding;
}
public void setDbfEncoding(String dbfEncoding) {
this.dbfEncoding = dbfEncoding;
}
public long getExecuteTimeout() {
return executeTimeout;
}
public void setExecuteTimeout(long executeTimeout) {
this.executeTimeout = executeTimeout;
}
public int getInsertBatchSize() {
return insertBatchSize;
}
public void setInsertBatchSize(int insertBatchSize) {
this.insertBatchSize = insertBatchSize;
}
}

42
clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java Normal file
View file

@ -0,0 +1,42 @@
package ru.spcex.clearing.dbf.export.services;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.export.logic.data.ISqlFilter;
import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
import ru.spcex.clearing.dbf.export.logic.stages.Stage;
import ru.spcex.clearing.dbf.export.properties.AProperties;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
@EnableScheduling
public class DBFExportService {
private final AProperties properties;
private final List<Stage> pipeline;
public DBFExportService(AProperties properties,
@Qualifier("pipeline") List<Stage> pipeline) {
this.properties = properties;
this.pipeline = pipeline;
}
@Scheduled(fixedDelay = 9999999999999999L)
public void run() {
Map<Table, ISqlFilter> tablesForExport = new HashMap<>();
for (Table table : Table.values()) tablesForExport.put(table, () -> "");
ResultContainer resultContainer = ResultContainer.createNewTask(tablesForExport);
for (Stage stage : pipeline) {
StageResult result = stage.process(resultContainer);
if (Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result))
break;
}
}
}

12
clearing-parent/dbf-export/src/main/resources/dbf_export.properties Normal file
View file

@ -0,0 +1,12 @@
db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres
db.driver=org.postgresql.Driver
db.login=clearing
db.password=Aa111111
dbf.check-source-timeout=300000000
dbf.execute-timeout=300000000
dbf.encoding=cp866
dbf.insert-batch-size=100
dbf.delete-src-files=false
dbf.src-dir=D:\\dbf\\

37
clearing-parent/dbf-export/src/main/resources/logback.xml Normal file
View file

@ -0,0 +1,37 @@
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<charset>UTF-8</charset>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>../logs/dbf_export.log</file>
<encoder>
<charset>UTF-8</charset>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>
../logs/dbf_export.%i.log
</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>10</maxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>500MB</maxFileSize>
</triggeringPolicy>
</appender>
<root level="warn">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE"/>
</root>
<logger name="ru.spcex" level="debug" additivity="false">
<appender-ref ref="FILE"/>
<appender-ref ref="CONSOLE"/>
</logger>
</configuration>

1
clearing-parent/pom.xml
View file

@ -22,6 +22,7 @@
<module>storage</module>
<module>db-scripts</module>
<module>dbf-loader</module>
<module>dbf-export</module>
</modules>
<properties>