diff --git a/clearing-parent/dbf-export/pom.xml b/clearing-parent/dbf-export/pom.xml new file mode 100644 index 000000000..093bb2711 --- /dev/null +++ b/clearing-parent/dbf-export/pom.xml @@ -0,0 +1,86 @@ + + + 4.0.0 + + dbf-export + dbf-export + DBF exporter for DBF files + SPCEX-1.0.0.0 + + + clearing-parent + ru.spcex.clearing + SPCEX-1.0.0.0 + + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-autoconfigure + + + + + org.springframework + spring-jdbc + + + com.mchange + c3p0 + 0.9.5.2 + + + org.postgresql + postgresql + + + + + com.github.albfernandez + javadbf + 1.13.1 + + + + + + jar/${project.artifactId} + + + org.apache.maven.plugins + maven-jar-plugin + + + + + ru.spcex.clearing.dbf.export.DBFLoaderApplication + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + + + true + exec + + + + + + \ No newline at end of file diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/DBFExportApplication.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/DBFExportApplication.java new file mode 100644 index 000000000..4d2684252 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/DBFExportApplication.java @@ -0,0 +1,18 @@ +package ru.spcex.clearing.dbf.export; + +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; + +@SpringBootApplication +public class DBFExportApplication { + public static void main(String[] args) { + try { + SpringApplicationBuilder builder = new SpringApplicationBuilder(DBFExportApplication.class); + builder.run(args); + } catch (Throwable e) { + LoggerFactory.getLogger(DBFExportApplication.class).error("DBF-Loader start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage()); + System.exit(-1); + } + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java new file mode 100644 index 000000000..44b4d3c5d --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java @@ -0,0 +1,54 @@ +package ru.spcex.clearing.dbf.export.config; + +import com.mchange.v2.c3p0.ComboPooledDataSource; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import ru.spcex.clearing.dbf.export.logic.stages.ExportFromDB; +import ru.spcex.clearing.dbf.export.logic.stages.Stage; +import ru.spcex.clearing.dbf.export.properties.AProperties; + +import javax.sql.DataSource; +import java.beans.PropertyVetoException; +import java.util.LinkedList; +import java.util.List; + +@Configuration +@ComponentScan(basePackages = {"ru.spcex.clearing.dbf.export"}) +public class DBFExportConfig { + private final AProperties properties; + private final ApplicationContext context; + + public DBFExportConfig(@Qualifier("dbfLoaderProperties") AProperties properties, ApplicationContext context) { + this.properties = properties; + this.context = context; + } + + @Bean("dbfDataSource") + public DataSource dataSource() throws PropertyVetoException { + ComboPooledDataSource dataSource = new ComboPooledDataSource(); + dataSource.setDriverClass(properties.getDbDriver()); + dataSource.setJdbcUrl(properties.getJdbcUrl()); + dataSource.setUser(properties.getDbLogin()); + dataSource.setPassword(properties.getDbPassword()); + return dataSource; + } + + @Bean("dbfJdbcTemplate") + public JdbcTemplate jdbcTemplate(@Qualifier("dbfDataSource") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Bean("pipeline") + public List pipeline() { + List pipeline = new LinkedList<>(); + + pipeline.add(context.getBean(ExportFromDB.class)); + + return pipeline; + } + +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/exceptions/ConfigException.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/exceptions/ConfigException.java new file mode 100644 index 000000000..ce17d44a7 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/exceptions/ConfigException.java @@ -0,0 +1,6 @@ +package ru.spcex.clearing.dbf.export.exceptions; + +public class ConfigException extends RuntimeException { + public ConfigException(String msg) { super(msg); } + public ConfigException(String msg, Throwable cause) { super(msg, cause); } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ISqlFilter.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ISqlFilter.java new file mode 100644 index 000000000..ff4b61033 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ISqlFilter.java @@ -0,0 +1,8 @@ +package ru.spcex.clearing.dbf.export.logic.data; + +/** + * Фильтр записей для экспорта + */ +public interface ISqlFilter { + String getSqlCondition(); +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java new file mode 100644 index 000000000..85bc97d82 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java @@ -0,0 +1,34 @@ +package ru.spcex.clearing.dbf.export.logic.data; + +import ru.spcex.clearing.dbf.export.logic.data.enums.Table; + +import java.io.File; +import java.util.Map; + +/** + * Контейнер для передачи результата между стадиями + */ +public class ResultContainer { + private Map tablesForExport; + private Map filesForTables; + + protected ResultContainer() {} + + public static ResultContainer createNewTask(Map tablesForExport) { + ResultContainer container = new ResultContainer(); + container.tablesForExport = tablesForExport; + return container; + } + + public Map getTablesForExport() { + return tablesForExport; + } + + public Map getFilesForTables() { + return filesForTables; + } + + public void setFilesForTables(Map filesForTables) { + this.filesForTables = filesForTables; + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/ColumnType.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/ColumnType.java new file mode 100644 index 000000000..b6445dbe2 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/ColumnType.java @@ -0,0 +1,46 @@ +package ru.spcex.clearing.dbf.export.logic.data.enums; + +import com.linuxense.javadbf.DBFDataType; + +import java.sql.Types; + +/** + * Типы данных в таблицах. + * Ставит в соответствие типы PostGRE и DBF + */ +public enum ColumnType { + VARCHAR(DBFDataType.CHARACTER, Types.VARCHAR), + CHARACTER(DBFDataType.CHARACTER, Types.CHAR), + NUMERIC(DBFDataType.NUMERIC, Types.NUMERIC), + DATE(DBFDataType.DATE, Types.DATE); + + private final DBFDataType dbfType; + private final int sqlType; + + ColumnType(DBFDataType dbfType, int postgreSqlType) { + this.dbfType = dbfType; + this.sqlType = postgreSqlType; + } + + public DBFDataType getDbfType() { + return dbfType; + } + + public int getSqlType() { + return sqlType; + } + + public static ColumnType getForSQLType(int sqlType) { + for (ColumnType columnType : values()) { + if (columnType.sqlType == sqlType) return columnType; + } + return null; + } + + public static ColumnType getForDBFType(DBFDataType dbfType) { + for (ColumnType columnType : values()) { + if (columnType.dbfType == dbfType) return columnType; + } + return null; + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/StageResult.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/StageResult.java new file mode 100644 index 000000000..76e9be113 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/StageResult.java @@ -0,0 +1,7 @@ +package ru.spcex.clearing.dbf.export.logic.data.enums; + +public enum StageResult { + OK, + ERROR, + COMPLETE +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/Table.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/Table.java new file mode 100644 index 000000000..6ce65dcec --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/Table.java @@ -0,0 +1,39 @@ +package ru.spcex.clearing.dbf.export.logic.data.enums; + +public enum Table { + DF_01("DF-01"), + DF_02("DF-02"), + DF_03("DF-03"), + DF_04("DF-04"), + DF_05("DF-05"), + DF_06("DF-06"); + + private final String filePrefix; + + Table(String prefix) { + this.filePrefix = prefix; + } + + public boolean fileForThisTable(String filename) { + return filename != null && filename.startsWith(filePrefix); + } + + public static Table getTableForFilename(String filename) { + for (Table table : Table.values()) { + if (table.fileForThisTable(filename)) return table; + } + return null; + } + + public static Table tableForName(String name) { + for (Table table : values()) { + if (name.equalsIgnoreCase(table.name())) + return table; + } + return null; + } + + public String getFilePrefix() { + return filePrefix; + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java new file mode 100644 index 000000000..0d4ca0194 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java @@ -0,0 +1,53 @@ +package ru.spcex.clearing.dbf.export.logic.stages; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.stereotype.Component; +import ru.spcex.clearing.dbf.export.exceptions.ConfigException; +import ru.spcex.clearing.dbf.export.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.export.logic.data.enums.Table; +import ru.spcex.clearing.dbf.export.properties.AProperties; + +import java.io.File; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; + +/** + * Создание DBF файлов + */ +@Component +public class CreateDBFFiles extends Stage implements InitializingBean { + private final AProperties properties; + + public CreateDBFFiles(AProperties properties) { + this.properties = properties; + } + + @Override + public StageResult process(ResultContainer resultContainer) { + Map dbfFiles = new HashMap<>(); + for (Table table : resultContainer.getTablesForExport().keySet()) { + File dbfFile = new File(table.getFilePrefix() + "_" + LocalDateTime.now()); + } + return StageResult.OK; + } + + /** + * 1. Собирает из БД названия столбцов для дальнейшей валидации + * 2. Проверяет кодировку из настройки dbf.encoding + */ + @Override + public void afterPropertiesSet() throws Exception { + initDBFCharset(); + } + + private void initDBFCharset() { + try { + Charset.forName(properties.getDbfEncoding()); + } catch (Exception e) { + throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e); + } + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java new file mode 100644 index 000000000..97a36190f --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java @@ -0,0 +1,29 @@ +package ru.spcex.clearing.dbf.export.logic.stages; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import ru.spcex.clearing.dbf.export.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.export.properties.AProperties; + +/** + * Выгрузка данных из базы и их запись + */ +@Component +public class ExportFromDB extends Stage { + private final AProperties properties; + private final JdbcTemplate dbfJdbcTemplate; + + public ExportFromDB(@Qualifier("dbfLoaderProperties") AProperties properties, + @Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) { + this.properties = properties; + this.dbfJdbcTemplate = dbfJdbcTemplate; + } + + @Override + public StageResult process(ResultContainer resultContainer) { + + return StageResult.OK; + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/Stage.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/Stage.java new file mode 100644 index 000000000..2329dc21d --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/Stage.java @@ -0,0 +1,12 @@ +package ru.spcex.clearing.dbf.export.logic.stages; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ru.spcex.clearing.dbf.export.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult; + +public abstract class Stage { + protected Logger log = LoggerFactory.getLogger(getClass()); + + public abstract StageResult process(ResultContainer resultContainer); +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java new file mode 100644 index 000000000..497b90c39 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java @@ -0,0 +1,121 @@ +package ru.spcex.clearing.dbf.export.properties; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; + +@Component("dbfLoaderProperties") +@PropertySource(value = {"classpath:dbf_export.properties"}) +@PropertySource(value = {"file:dbf_export.properties"}, ignoreResourceNotFound = true) +public class AProperties { + + @Value("${db.jdbc-url}") + private String jdbcUrl; + + @Value("${db.driver}") + private String dbDriver; + + @Value("${db.login}") + private String dbLogin; + + @Value("${db.password}") + private String dbPassword; + + @Value("${dbf.src-dir}") + private String srcDir; + + @Value("${dbf.delete-src-files}") + private boolean deleteSrcFiles = true; + + @Value("${dbf.check-source-timeout}") + private long checkSourceTimeout; + + @Value("${dbf.execute-timeout}") + private long executeTimeout; + + @Value("${dbf.encoding}") + private String dbfEncoding; + + @Value("${dbf.insert-batch-size}") + private int insertBatchSize; + + public String getJdbcUrl() { + return jdbcUrl; + } + + public void setJdbcUrl(String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + } + + public String getDbLogin() { + return dbLogin; + } + + public void setDbLogin(String dbLogin) { + this.dbLogin = dbLogin; + } + + public String getDbPassword() { + return dbPassword; + } + + public void setDbPassword(String dbPassword) { + this.dbPassword = dbPassword; + } + + public String getDbDriver() { + return dbDriver; + } + + public void setDbDriver(String dbDriver) { + this.dbDriver = dbDriver; + } + + public String getSrcDir() { + return srcDir; + } + + public void setSrcDir(String srcDir) { + this.srcDir = srcDir; + } + + public long getCheckSourceTimeout() { + return checkSourceTimeout; + } + + public void setCheckSourceTimeout(long checkSourceTimeout) { + this.checkSourceTimeout = checkSourceTimeout; + } + + public boolean deleteSrcFiles() { + return deleteSrcFiles; + } + + public void setDeleteSrcFiles(boolean deleteSrcFiles) { + this.deleteSrcFiles = deleteSrcFiles; + } + + public String getDbfEncoding() { + return dbfEncoding; + } + + public void setDbfEncoding(String dbfEncoding) { + this.dbfEncoding = dbfEncoding; + } + + public long getExecuteTimeout() { + return executeTimeout; + } + + public void setExecuteTimeout(long executeTimeout) { + this.executeTimeout = executeTimeout; + } + + public int getInsertBatchSize() { + return insertBatchSize; + } + + public void setInsertBatchSize(int insertBatchSize) { + this.insertBatchSize = insertBatchSize; + } +} diff --git a/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java new file mode 100644 index 000000000..d6713d489 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java @@ -0,0 +1,42 @@ +package ru.spcex.clearing.dbf.export.services; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import ru.spcex.clearing.dbf.export.logic.data.ISqlFilter; +import ru.spcex.clearing.dbf.export.logic.data.ResultContainer; +import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult; +import ru.spcex.clearing.dbf.export.logic.data.enums.Table; +import ru.spcex.clearing.dbf.export.logic.stages.Stage; +import ru.spcex.clearing.dbf.export.properties.AProperties; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@EnableScheduling +public class DBFExportService { + private final AProperties properties; + private final List pipeline; + + public DBFExportService(AProperties properties, + @Qualifier("pipeline") List pipeline) { + this.properties = properties; + this.pipeline = pipeline; + } + + @Scheduled(fixedDelay = 9999999999999999L) + public void run() { + Map tablesForExport = new HashMap<>(); + for (Table table : Table.values()) tablesForExport.put(table, () -> ""); + ResultContainer resultContainer = ResultContainer.createNewTask(tablesForExport); + for (Stage stage : pipeline) { + StageResult result = stage.process(resultContainer); + if (Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result)) + break; + } + } +} diff --git a/clearing-parent/dbf-export/src/main/resources/dbf_export.properties b/clearing-parent/dbf-export/src/main/resources/dbf_export.properties new file mode 100644 index 000000000..0e309b572 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/resources/dbf_export.properties @@ -0,0 +1,12 @@ +db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres +db.driver=org.postgresql.Driver +db.login=clearing +db.password=Aa111111 + +dbf.check-source-timeout=300000000 +dbf.execute-timeout=300000000 +dbf.encoding=cp866 +dbf.insert-batch-size=100 + +dbf.delete-src-files=false +dbf.src-dir=D:\\dbf\\ diff --git a/clearing-parent/dbf-export/src/main/resources/logback.xml b/clearing-parent/dbf-export/src/main/resources/logback.xml new file mode 100644 index 000000000..33ca393d8 --- /dev/null +++ b/clearing-parent/dbf-export/src/main/resources/logback.xml @@ -0,0 +1,37 @@ + + + + + UTF-8 + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + ../logs/dbf_export.log + + UTF-8 + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + ../logs/dbf_export.%i.log + + 1 + 10 + + + 500MB + + + + + + + + + + + + + \ No newline at end of file diff --git a/clearing-parent/pom.xml b/clearing-parent/pom.xml index 2fc9ebc13..3c1783d7d 100644 --- a/clearing-parent/pom.xml +++ b/clearing-parent/pom.xml @@ -22,6 +22,7 @@ storage db-scripts dbf-loader + dbf-export