add dbf export module (in progress)

clearing-parent/dbf-export/pom.xml

@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>dbf-export</artifactId>
+    <name>dbf-export</name>
+    <description>DBF exporter for DBF files</description>
+    <version>SPCEX-1.0.0.0</version>
+
+    <parent>
+        <artifactId>clearing-parent</artifactId>
+        <groupId>ru.spcex.clearing</groupId>
+        <version>SPCEX-1.0.0.0</version>
+    </parent>
+
+    <dependencies>
+        <!-- Spring boot -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-autoconfigure</artifactId>
+        </dependency>
+
+        <!-- JDBC -->
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.mchange</groupId>
+            <artifactId>c3p0</artifactId>
+            <version>0.9.5.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+        </dependency>
+
+        <!-- DBF files -->
+        <dependency>
+            <groupId>com.github.albfernandez</groupId>
+            <artifactId>javadbf</artifactId>
+            <version>1.13.1</version>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <finalName>jar/${project.artifactId}</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <!--<classpathPrefix>../lib/</classpathPrefix>-->
+                            <mainClass>ru.spcex.clearing.dbf.export.DBFLoaderApplication</mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <addResources>true</addResources>
+                    <classifier>exec</classifier>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/DBFExportApplication.java

@@ -0,0 +1,18 @@
+package ru.spcex.clearing.dbf.export;
+
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+
+@SpringBootApplication
+public class DBFExportApplication {
+    public static void main(String[] args) {
+        try {
+            SpringApplicationBuilder builder = new SpringApplicationBuilder(DBFExportApplication.class);
+            builder.run(args);
+        } catch (Throwable e) {
+            LoggerFactory.getLogger(DBFExportApplication.class).error("DBF-Loader start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage());
+            System.exit(-1);
+        }
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java

@@ -0,0 +1,54 @@
+package ru.spcex.clearing.dbf.export.config;
+
+import com.mchange.v2.c3p0.ComboPooledDataSource;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+import ru.spcex.clearing.dbf.export.logic.stages.ExportFromDB;
+import ru.spcex.clearing.dbf.export.logic.stages.Stage;
+import ru.spcex.clearing.dbf.export.properties.AProperties;
+
+import javax.sql.DataSource;
+import java.beans.PropertyVetoException;
+import java.util.LinkedList;
+import java.util.List;
+
+@Configuration
+@ComponentScan(basePackages = {"ru.spcex.clearing.dbf.export"})
+public class DBFExportConfig {
+    private final AProperties properties;
+    private final ApplicationContext context;
+
+    public DBFExportConfig(@Qualifier("dbfLoaderProperties") AProperties properties, ApplicationContext context) {
+        this.properties = properties;
+        this.context = context;
+    }
+
+    @Bean("dbfDataSource")
+    public DataSource dataSource() throws PropertyVetoException {
+        ComboPooledDataSource dataSource = new ComboPooledDataSource();
+        dataSource.setDriverClass(properties.getDbDriver());
+        dataSource.setJdbcUrl(properties.getJdbcUrl());
+        dataSource.setUser(properties.getDbLogin());
+        dataSource.setPassword(properties.getDbPassword());
+        return dataSource;
+    }
+
+    @Bean("dbfJdbcTemplate")
+    public JdbcTemplate jdbcTemplate(@Qualifier("dbfDataSource") DataSource dataSource) {
+        return new JdbcTemplate(dataSource);
+    }
+
+    @Bean("pipeline")
+    public List<Stage> pipeline() {
+        List<Stage> pipeline = new LinkedList<>();
+
+        pipeline.add(context.getBean(ExportFromDB.class));
+
+        return pipeline;
+    }
+
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/exceptions/ConfigException.java

@@ -0,0 +1,6 @@
+package ru.spcex.clearing.dbf.export.exceptions;
+
+public class ConfigException extends RuntimeException {
+    public ConfigException(String msg) { super(msg); }
+    public ConfigException(String msg, Throwable cause) { super(msg, cause); }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ISqlFilter.java

@@ -0,0 +1,8 @@
+package ru.spcex.clearing.dbf.export.logic.data;
+
+/**
+ * Фильтр записей для экспорта
+ */
+public interface ISqlFilter {
+    String getSqlCondition();
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java

@@ -0,0 +1,34 @@
+package ru.spcex.clearing.dbf.export.logic.data;
+
+import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
+
+import java.io.File;
+import java.util.Map;
+
+/**
+ * Контейнер для передачи результата между стадиями
+ */
+public class ResultContainer {
+    private Map<Table, ISqlFilter> tablesForExport;
+    private Map<Table, File> filesForTables;
+
+    protected ResultContainer() {}
+
+    public static ResultContainer createNewTask(Map<Table, ISqlFilter> tablesForExport) {
+        ResultContainer container = new ResultContainer();
+        container.tablesForExport = tablesForExport;
+        return container;
+    }
+
+    public Map<Table, ISqlFilter> getTablesForExport() {
+        return tablesForExport;
+    }
+
+    public Map<Table, File> getFilesForTables() {
+        return filesForTables;
+    }
+
+    public void setFilesForTables(Map<Table, File> filesForTables) {
+        this.filesForTables = filesForTables;
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/ColumnType.java

@@ -0,0 +1,46 @@
+package ru.spcex.clearing.dbf.export.logic.data.enums;
+
+import com.linuxense.javadbf.DBFDataType;
+
+import java.sql.Types;
+
+/**
+ * Типы данных в таблицах.
+ * Ставит в соответствие типы PostGRE и DBF
+ */
+public enum ColumnType {
+    VARCHAR(DBFDataType.CHARACTER, Types.VARCHAR),
+    CHARACTER(DBFDataType.CHARACTER, Types.CHAR),
+    NUMERIC(DBFDataType.NUMERIC, Types.NUMERIC),
+    DATE(DBFDataType.DATE, Types.DATE);
+
+    private final DBFDataType dbfType;
+    private final int sqlType;
+
+    ColumnType(DBFDataType dbfType, int postgreSqlType) {
+        this.dbfType = dbfType;
+        this.sqlType = postgreSqlType;
+    }
+
+    public DBFDataType getDbfType() {
+        return dbfType;
+    }
+
+    public int getSqlType() {
+        return sqlType;
+    }
+
+    public static ColumnType getForSQLType(int sqlType) {
+        for (ColumnType columnType : values()) {
+            if (columnType.sqlType == sqlType) return columnType;
+        }
+        return null;
+    }
+
+    public static ColumnType getForDBFType(DBFDataType dbfType) {
+        for (ColumnType columnType : values()) {
+            if (columnType.dbfType == dbfType) return columnType;
+        }
+        return null;
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/StageResult.java

@@ -0,0 +1,7 @@
+package ru.spcex.clearing.dbf.export.logic.data.enums;
+
+public enum StageResult {
+    OK,
+    ERROR,
+    COMPLETE
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/enums/Table.java

@@ -0,0 +1,39 @@
+package ru.spcex.clearing.dbf.export.logic.data.enums;
+
+public enum Table {
+    DF_01("DF-01"),
+    DF_02("DF-02"),
+    DF_03("DF-03"),
+    DF_04("DF-04"),
+    DF_05("DF-05"),
+    DF_06("DF-06");
+
+    private final String filePrefix;
+
+    Table(String prefix) {
+        this.filePrefix = prefix;
+    }
+
+    public boolean fileForThisTable(String filename) {
+        return filename != null && filename.startsWith(filePrefix);
+    }
+
+    public static Table getTableForFilename(String filename) {
+        for (Table table : Table.values()) {
+            if (table.fileForThisTable(filename)) return table;
+        }
+        return null;
+    }
+
+    public static Table tableForName(String name) {
+        for (Table table : values()) {
+            if (name.equalsIgnoreCase(table.name()))
+                return table;
+        }
+        return null;
+    }
+
+    public String getFilePrefix() {
+        return filePrefix;
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java

@@ -0,0 +1,53 @@
+package ru.spcex.clearing.dbf.export.logic.stages;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Component;
+import ru.spcex.clearing.dbf.export.exceptions.ConfigException;
+import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
+import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
+import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.export.properties.AProperties;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Создание DBF файлов
+ */
+@Component
+public class CreateDBFFiles extends Stage implements InitializingBean {
+    private final AProperties properties;
+
+    public CreateDBFFiles(AProperties properties) {
+        this.properties = properties;
+    }
+
+    @Override
+    public StageResult process(ResultContainer resultContainer) {
+        Map<Table, File> dbfFiles = new HashMap<>();
+        for (Table table : resultContainer.getTablesForExport().keySet()) {
+            File dbfFile = new File(table.getFilePrefix() + "_" + LocalDateTime.now());
+        }
+        return StageResult.OK;
+    }
+
+    /**
+     * 1. Собирает из БД названия столбцов для дальнейшей валидации
+     * 2. Проверяет кодировку из настройки dbf.encoding
+     */
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        initDBFCharset();
+    }
+
+    private void initDBFCharset() {
+        try {
+            Charset.forName(properties.getDbfEncoding());
+        } catch (Exception e) {
+            throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
+        }
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java

@@ -0,0 +1,29 @@
+package ru.spcex.clearing.dbf.export.logic.stages;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Component;
+import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
+import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
+import ru.spcex.clearing.dbf.export.properties.AProperties;
+
+/**
+ * Выгрузка данных из базы и их запись
+ */
+@Component
+public class ExportFromDB extends Stage {
+    private final AProperties properties;
+    private final JdbcTemplate dbfJdbcTemplate;
+
+    public ExportFromDB(@Qualifier("dbfLoaderProperties") AProperties properties,
+                        @Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) {
+        this.properties = properties;
+        this.dbfJdbcTemplate = dbfJdbcTemplate;
+    }
+
+    @Override
+    public StageResult process(ResultContainer resultContainer) {
+
+        return StageResult.OK;
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/Stage.java

@@ -0,0 +1,12 @@
+package ru.spcex.clearing.dbf.export.logic.stages;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
+import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
+
+public abstract class Stage {
+    protected Logger log = LoggerFactory.getLogger(getClass());
+
+    public abstract StageResult process(ResultContainer resultContainer);
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java

@@ -0,0 +1,121 @@
+package ru.spcex.clearing.dbf.export.properties;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
+
+@Component("dbfLoaderProperties")
+@PropertySource(value = {"classpath:dbf_export.properties"})
+@PropertySource(value = {"file:dbf_export.properties"}, ignoreResourceNotFound = true)
+public class AProperties {
+
+    @Value("${db.jdbc-url}")
+    private String jdbcUrl;
+
+    @Value("${db.driver}")
+    private String dbDriver;
+
+    @Value("${db.login}")
+    private String dbLogin;
+
+    @Value("${db.password}")
+    private String dbPassword;
+
+    @Value("${dbf.src-dir}")
+    private String srcDir;
+
+    @Value("${dbf.delete-src-files}")
+    private boolean deleteSrcFiles = true;
+
+    @Value("${dbf.check-source-timeout}")
+    private long checkSourceTimeout;
+
+    @Value("${dbf.execute-timeout}")
+    private long executeTimeout;
+
+    @Value("${dbf.encoding}")
+    private String dbfEncoding;
+
+    @Value("${dbf.insert-batch-size}")
+    private int insertBatchSize;
+
+    public String getJdbcUrl() {
+        return jdbcUrl;
+    }
+
+    public void setJdbcUrl(String jdbcUrl) {
+        this.jdbcUrl = jdbcUrl;
+    }
+
+    public String getDbLogin() {
+        return dbLogin;
+    }
+
+    public void setDbLogin(String dbLogin) {
+        this.dbLogin = dbLogin;
+    }
+
+    public String getDbPassword() {
+        return dbPassword;
+    }
+
+    public void setDbPassword(String dbPassword) {
+        this.dbPassword = dbPassword;
+    }
+
+    public String getDbDriver() {
+        return dbDriver;
+    }
+
+    public void setDbDriver(String dbDriver) {
+        this.dbDriver = dbDriver;
+    }
+
+    public String getSrcDir() {
+        return srcDir;
+    }
+
+    public void setSrcDir(String srcDir) {
+        this.srcDir = srcDir;
+    }
+
+    public long getCheckSourceTimeout() {
+        return checkSourceTimeout;
+    }
+
+    public void setCheckSourceTimeout(long checkSourceTimeout) {
+        this.checkSourceTimeout = checkSourceTimeout;
+    }
+
+    public boolean deleteSrcFiles() {
+        return deleteSrcFiles;
+    }
+
+    public void setDeleteSrcFiles(boolean deleteSrcFiles) {
+        this.deleteSrcFiles = deleteSrcFiles;
+    }
+
+    public String getDbfEncoding() {
+        return dbfEncoding;
+    }
+
+    public void setDbfEncoding(String dbfEncoding) {
+        this.dbfEncoding = dbfEncoding;
+    }
+
+    public long getExecuteTimeout() {
+        return executeTimeout;
+    }
+
+    public void setExecuteTimeout(long executeTimeout) {
+        this.executeTimeout = executeTimeout;
+    }
+
+    public int getInsertBatchSize() {
+        return insertBatchSize;
+    }
+
+    public void setInsertBatchSize(int insertBatchSize) {
+        this.insertBatchSize = insertBatchSize;
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java

@@ -0,0 +1,42 @@
+package ru.spcex.clearing.dbf.export.services;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import ru.spcex.clearing.dbf.export.logic.data.ISqlFilter;
+import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
+import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
+import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.export.logic.stages.Stage;
+import ru.spcex.clearing.dbf.export.properties.AProperties;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+@EnableScheduling
+public class DBFExportService {
+    private final AProperties properties;
+    private final List<Stage> pipeline;
+
+    public DBFExportService(AProperties properties,
+                            @Qualifier("pipeline") List<Stage> pipeline) {
+        this.properties = properties;
+        this.pipeline = pipeline;
+    }
+
+    @Scheduled(fixedDelay = 9999999999999999L)
+    public void run() {
+        Map<Table, ISqlFilter> tablesForExport = new HashMap<>();
+        for (Table table : Table.values()) tablesForExport.put(table, () -> "");
+        ResultContainer resultContainer = ResultContainer.createNewTask(tablesForExport);
+        for (Stage stage : pipeline) {
+            StageResult result = stage.process(resultContainer);
+            if (Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result))
+                break;
+        }
+    }
+}

clearing-parent/dbf-export/src/main/resources/dbf_export.properties

@@ -0,0 +1,12 @@
+db.jdbc-url=jdbc:postgresql://10.200.200.133:5432/postgres
+db.driver=org.postgresql.Driver
+db.login=clearing
+db.password=Aa111111
+
+dbf.check-source-timeout=300000000
+dbf.execute-timeout=300000000
+dbf.encoding=cp866
+dbf.insert-batch-size=100
+
+dbf.delete-src-files=false
+dbf.src-dir=D:\\dbf\\

clearing-parent/dbf-export/src/main/resources/logback.xml

@@ -0,0 +1,37 @@
+<configuration>
+
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <charset>UTF-8</charset>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>../logs/dbf_export.log</file>
+        <encoder>
+            <charset>UTF-8</charset>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>
+                ../logs/dbf_export.%i.log
+            </fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>500MB</maxFileSize>
+        </triggeringPolicy>
+    </appender>
+
+    <root level="warn">
+        <appender-ref ref="CONSOLE"/>
+        <appender-ref ref="FILE"/>
+    </root>
+
+    <logger name="ru.spcex" level="debug" additivity="false">
+        <appender-ref ref="FILE"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+</configuration>

clearing-parent/pom.xml

@@ -22,6 +22,7 @@
         <module>storage</module>
         <module>db-scripts</module>
         <module>dbf-loader</module>
+        <module>dbf-export</module>
     </modules>
 
     <properties>