add dbf export module

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/config/DBFExportConfig.java

@@ -8,6 +8,7 @@ import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.jdbc.core.JdbcTemplate;
 import ru.spcex.clearing.dbf.export.logic.stages.ExportFromDB;
+import ru.spcex.clearing.dbf.export.logic.stages.PrepareDBFFiles;
 import ru.spcex.clearing.dbf.export.logic.stages.Stage;
 import ru.spcex.clearing.dbf.export.properties.AProperties;
 
@@ -22,7 +23,7 @@ public class DBFExportConfig {
     private final AProperties properties;
     private final ApplicationContext context;
 
-    public DBFExportConfig(@Qualifier("dbfLoaderProperties") AProperties properties, ApplicationContext context) {
+    public DBFExportConfig(@Qualifier("dbfExportProperties") AProperties properties, ApplicationContext context) {
         this.properties = properties;
         this.context = context;
     }
@@ -46,6 +47,7 @@ public class DBFExportConfig {
     public List<Stage> pipeline() {
         List<Stage> pipeline = new LinkedList<>();
 
+        pipeline.add(context.getBean(PrepareDBFFiles.class));
         pipeline.add(context.getBean(ExportFromDB.class));
 
         return pipeline;

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/data/ResultContainer.java

@@ -31,4 +31,5 @@ public class ResultContainer {
     public void setFilesForTables(Map<Table, File> filesForTables) {
         this.filesForTables = filesForTables;
     }
+
 }

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/CreateDBFFiles.java

@@ -1,53 +0,0 @@
-package ru.spcex.clearing.dbf.export.logic.stages;
-
-import org.springframework.beans.factory.InitializingBean;
-import org.springframework.stereotype.Component;
-import ru.spcex.clearing.dbf.export.exceptions.ConfigException;
-import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
-import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
-import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
-import ru.spcex.clearing.dbf.export.properties.AProperties;
-
-import java.io.File;
-import java.nio.charset.Charset;
-import java.time.LocalDateTime;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Создание DBF файлов
- */
-@Component
-public class CreateDBFFiles extends Stage implements InitializingBean {
-    private final AProperties properties;
-
-    public CreateDBFFiles(AProperties properties) {
-        this.properties = properties;
-    }
-
-    @Override
-    public StageResult process(ResultContainer resultContainer) {
-        Map<Table, File> dbfFiles = new HashMap<>();
-        for (Table table : resultContainer.getTablesForExport().keySet()) {
-            File dbfFile = new File(table.getFilePrefix() + "_" + LocalDateTime.now());
-        }
-        return StageResult.OK;
-    }
-
-    /**
-     * 1. Собирает из БД названия столбцов для дальнейшей валидации
-     * 2. Проверяет кодировку из настройки dbf.encoding
-     */
-    @Override
-    public void afterPropertiesSet() throws Exception {
-        initDBFCharset();
-    }
-
-    private void initDBFCharset() {
-        try {
-            Charset.forName(properties.getDbfEncoding());
-        } catch (Exception e) {
-            throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
-        }
-    }
-}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/ExportFromDB.java

@@ -1,21 +1,44 @@
 package ru.spcex.clearing.dbf.export.logic.stages;
 
+import com.linuxense.javadbf.DBFDataType;
+import com.linuxense.javadbf.DBFException;
+import com.linuxense.javadbf.DBFField;
+import com.linuxense.javadbf.DBFWriter;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.support.JdbcUtils;
+import org.springframework.jdbc.support.MetaDataAccessException;
 import org.springframework.stereotype.Component;
+import ru.spcex.clearing.dbf.export.exceptions.ConfigException;
+import ru.spcex.clearing.dbf.export.logic.data.ISqlFilter;
 import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
+import ru.spcex.clearing.dbf.export.logic.data.enums.ColumnType;
 import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
+import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
 import ru.spcex.clearing.dbf.export.properties.AProperties;
 
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.charset.Charset;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.util.*;
+
 /**
  * Выгрузка данных из базы и их запись
  */
 @Component
-public class ExportFromDB extends Stage {
+public class ExportFromDB extends Stage implements InitializingBean {
     private final AProperties properties;
     private final JdbcTemplate dbfJdbcTemplate;
+    private final Map<Table, DBFField[]> dbfFieldsForTable = new HashMap<>();
 
-    public ExportFromDB(@Qualifier("dbfLoaderProperties") AProperties properties,
+    private Charset dbfCharset;
+
+    public ExportFromDB(@Qualifier("dbfExportProperties") AProperties properties,
                         @Qualifier("dbfJdbcTemplate") JdbcTemplate dbfJdbcTemplate) {
         this.properties = properties;
         this.dbfJdbcTemplate = dbfJdbcTemplate;
@@ -23,7 +46,104 @@ public class ExportFromDB extends Stage {
 
     @Override
     public StageResult process(ResultContainer resultContainer) {
+        for (Table table : resultContainer.getTablesForExport().keySet()) {
+            ISqlFilter sqlFilter = resultContainer.getTablesForExport().get(table);
+            String sqlCondition = sqlFilter != null ? " where " + sqlFilter : "";
+
+            String sql = "select * from " + table + sqlCondition;
+            List<Map<String, Object>> recordsFromDB = dbfJdbcTemplate.queryForList(sql);
+
+            File dbfFile = resultContainer.getFilesForTables().get(table);
+            DBFField[] dbfFields = dbfFieldsForTable.get(table);
+
+            try {
+                boolean createOk = dbfFile.createNewFile();
+                if (!createOk) throw new IOException("Can't create " + dbfFile.getName() + " file");
+            } catch (IOException e) {
+                log.error("Can't create file " + dbfFile + " for table " + table + ". Table was skipped.", e);
+                continue;
+            }
+
+            try (DBFWriter dbfWriter = new DBFWriter(dbfFile, dbfCharset)) {
+                dbfWriter.setFields(dbfFields);
+                for (Map<String, Object> record : recordsFromDB) {
+                    int columnCount = dbfFields.length;
+                    Object[] values = new Object[columnCount];
+                    for (int columnIdx = 0; columnIdx < columnCount; columnIdx++) {
+                        DBFDataType currDBFType = dbfFields[columnIdx].getType();
+                        Object currColumn = record.get(dbfFields[columnIdx].getName());
+
+                        if (currDBFType == DBFDataType.LOGICAL) {
+                            values[columnIdx] = Boolean.parseBoolean(String.valueOf(currColumn));
+
+                        } else if (currDBFType == DBFDataType.DATE) {
+                            LocalDate date = LocalDate.parse(String.valueOf(currColumn));
+                            values[columnIdx] = new Date(date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli());
+
+                        } else if (currDBFType == DBFDataType.NUMERIC || currDBFType == DBFDataType.FLOATING_POINT)  {
+                            BigDecimal bigDecimal = new BigDecimal(String.valueOf(currColumn));
+                            values[columnIdx] = bigDecimal;
+
+                        } else {
+                            values[columnIdx] = String.valueOf(currColumn);
+
+                        }
+                    }
+                    dbfWriter.addRecord(values);
+                }
+            } catch (DBFException e) {
+                log.error("Can't export table " + table + " to file " + dbfFile + ". Table was skipped.", e);
+            }
+        }
 
         return StageResult.OK;
     }
+
+
+    /**
+     * 1. Собирает структуру БД для дальнейшей записи DBF файлов
+     * 2. Проверяет кодировку из настройки dbf.encoding
+     */
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        initDBStructure();
+        initDBFCharset();
+    }
+
+    private void initDBStructure() throws MetaDataAccessException {
+        boolean ok = JdbcUtils.extractDatabaseMetaData(Objects.requireNonNull(dbfJdbcTemplate.getDataSource()),
+                dbMeta -> {
+                    List<String> tableNamesFromResultSet = new LinkedList<>();
+                    ResultSet tableNamesRS = dbMeta.getTables(null, null, "%", new String[]{"TABLE"});
+                    while (tableNamesRS.next()) {
+                        tableNamesFromResultSet.add(tableNamesRS.getString("TABLE_NAME"));
+                    }
+                    for (String tableNameFromResultSet : tableNamesFromResultSet) {
+                        Table currTable = Table.tableForName(tableNameFromResultSet);
+                        if (currTable == null) continue;
+                        List<DBFField> dbfFields = new ArrayList<>();
+                        ResultSet columnNamesRS = dbMeta.getColumns(null, null, tableNameFromResultSet, null);
+                        while (columnNamesRS.next()) {
+                            String name = columnNamesRS.getString("COLUMN_NAME");
+                            ColumnType type = ColumnType.getForSQLType(columnNamesRS.getInt("DATA_TYPE"));
+                            if (type == null) throw new ConfigException("Unsupported ColumnType from DB");
+                            int length = columnNamesRS.getInt("COLUMN_SIZE");
+                            if (type == ColumnType.DATE) length = 8;
+                            int decimalCount = columnNamesRS.getInt("DECIMAL_DIGITS");
+                            DBFField dbfField = new DBFField(name.toUpperCase(), type.getDbfType(), length, decimalCount);
+                            dbfFields.add(dbfField);
+                        }
+                        dbfFieldsForTable.put(currTable, dbfFields.toArray(DBFField[]::new));
+                    }
+                    return true;
+                });
+    }
+
+    private void initDBFCharset() {
+        try {
+            dbfCharset = Charset.forName(properties.getDbfEncoding());
+        } catch (Exception e) {
+            throw new ConfigException("В properties файле содержится неизвестная кодировка: " + properties.getDbfEncoding(), e);
+        }
+    }
 }

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/logic/stages/PrepareDBFFiles.java

@@ -0,0 +1,69 @@
+package ru.spcex.clearing.dbf.export.logic.stages;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+import ru.spcex.clearing.dbf.export.logic.data.ResultContainer;
+import ru.spcex.clearing.dbf.export.logic.data.enums.StageResult;
+import ru.spcex.clearing.dbf.export.logic.data.enums.Table;
+import ru.spcex.clearing.dbf.export.properties.AProperties;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Создание DBF файлов
+ */
+@Component
+public class PrepareDBFFiles extends Stage implements InitializingBean {
+    private final AProperties properties;
+    private String outDirWithDateTime;
+
+    public PrepareDBFFiles(@Qualifier("dbfExportProperties") AProperties properties) {
+        this.properties = properties;
+    }
+
+    @Override
+    public StageResult process(ResultContainer resultContainer) {
+        Map<Table, File> dbfFiles = new HashMap<>();
+        for (Table table : resultContainer.getTablesForExport().keySet()) {
+            File dbfFile = new File(String.format("%s%s%s.dbf",
+                    outDirWithDateTime,
+                    File.separator,
+                    table.getFilePrefix()));
+            try {
+                if (dbfFile.exists()) {
+                    boolean deleteOk = dbfFile.delete();
+                    if (!deleteOk) throw new IOException("Can't overwrite " + dbfFile.getName() + " file");
+                }
+
+                dbfFiles.put(table, dbfFile);
+            } catch (IOException e) {
+                log.error("Can't create " + dbfFile + ". Table was skipped", e);
+                resultContainer.getTablesForExport().remove(table);
+            }
+        }
+        resultContainer.setFilesForTables(dbfFiles);
+        return StageResult.OK;
+    }
+
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        String outDirPath = properties.getOutDir();
+        File outDir = new File(String.format("%s%s%s",
+                outDirPath,
+                File.separator,
+                LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmmss"))));
+        if (outDir.exists()) throw new IOException("Output dir " + outDir + " already exist.");
+
+        boolean createDirOk = outDir.mkdirs();
+        if (!createDirOk) throw new IOException("Can't create output dir " + outDir);
+
+        outDirWithDateTime = outDir.getAbsolutePath();
+    }
+}

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/properties/AProperties.java

@@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.PropertySource;
 import org.springframework.stereotype.Component;
 
-@Component("dbfLoaderProperties")
+@Component("dbfExportProperties")
 @PropertySource(value = {"classpath:dbf_export.properties"})
 @PropertySource(value = {"file:dbf_export.properties"}, ignoreResourceNotFound = true)
 public class AProperties {
@@ -21,24 +21,12 @@ public class AProperties {
     @Value("${db.password}")
     private String dbPassword;
 
-    @Value("${dbf.src-dir}")
+    @Value("${dbf.out-dir}")
-    private String srcDir;
+    private String outDir;
-
-    @Value("${dbf.delete-src-files}")
-    private boolean deleteSrcFiles = true;
-
-    @Value("${dbf.check-source-timeout}")
-    private long checkSourceTimeout;
-
-    @Value("${dbf.execute-timeout}")
-    private long executeTimeout;
 
     @Value("${dbf.encoding}")
     private String dbfEncoding;
 
-    @Value("${dbf.insert-batch-size}")
-    private int insertBatchSize;
-
     public String getJdbcUrl() {
         return jdbcUrl;
     }
@@ -71,28 +59,12 @@ public class AProperties {
         this.dbDriver = dbDriver;
     }
 
-    public String getSrcDir() {
+    public String getOutDir() {
-        return srcDir;
+        return outDir;
     }
 
-    public void setSrcDir(String srcDir) {
+    public void setOutDir(String outDir) {
-        this.srcDir = srcDir;
+        this.outDir = outDir;
-    }
-
-    public long getCheckSourceTimeout() {
-        return checkSourceTimeout;
-    }
-
-    public void setCheckSourceTimeout(long checkSourceTimeout) {
-        this.checkSourceTimeout = checkSourceTimeout;
-    }
-
-    public boolean deleteSrcFiles() {
-        return deleteSrcFiles;
-    }
-
-    public void setDeleteSrcFiles(boolean deleteSrcFiles) {
-        this.deleteSrcFiles = deleteSrcFiles;
     }
 
     public String getDbfEncoding() {
@@ -102,20 +74,4 @@ public class AProperties {
     public void setDbfEncoding(String dbfEncoding) {
         this.dbfEncoding = dbfEncoding;
     }
-
-    public long getExecuteTimeout() {
-        return executeTimeout;
-    }
-
-    public void setExecuteTimeout(long executeTimeout) {
-        this.executeTimeout = executeTimeout;
-    }
-
-    public int getInsertBatchSize() {
-        return insertBatchSize;
-    }
-
-    public void setInsertBatchSize(int insertBatchSize) {
-        this.insertBatchSize = insertBatchSize;
-    }
 }

clearing-parent/dbf-export/src/main/java/ru/spcex/clearing/dbf/export/services/DBFExportService.java

@@ -31,7 +31,7 @@ public class DBFExportService {
     @Scheduled(fixedDelay = 9999999999999999L)
     public void run() {
         Map<Table, ISqlFilter> tablesForExport = new HashMap<>();
-        for (Table table : Table.values()) tablesForExport.put(table, () -> "");
+        for (Table table : Table.values()) tablesForExport.put(table, null);
         ResultContainer resultContainer = ResultContainer.createNewTask(tablesForExport);
         for (Stage stage : pipeline) {
             StageResult result = stage.process(resultContainer);

clearing-parent/dbf-export/src/main/resources/dbf_export.properties

@@ -3,10 +3,6 @@ db.driver=org.postgresql.Driver
 db.login=clearing
 db.password=Aa111111
 
-dbf.check-source-timeout=300000000
-dbf.execute-timeout=300000000
 dbf.encoding=cp866
-dbf.insert-batch-size=100
 
-dbf.delete-src-files=false
+dbf.out-dir=D:\\dbf\\out
-dbf.src-dir=D:\\dbf\\