From 700993340943f3793c8ab8a993d4d5a6b370d862 Mon Sep 17 00:00:00 2001 From: akulikov Date: Thu, 11 Aug 2022 19:46:48 +0300 Subject: [PATCH] dbf exporter and fix dbf importer --- clearing-parent/dbf-exporter/pom.xml | 10 ---------- .../{DBFExportConfig.java => DBFExporterConfig.java} | 10 +++++++--- .../dbf/exporter/logic/stages/PrepareDBFFile.java | 10 +++++++--- .../clearing/dbf/exporter/logic/stages/Processor.java | 2 +- .../dbf/exporter/services/DBFExportService.java | 8 +++++++- .../src/main/resources/application.properties | 2 +- .../dbf/importer/config/DBFImporterConfig.java | 6 +++++- .../clearing/dbf/importer/logic/stages/Processor.java | 2 +- .../dbf/importer/services/DBFImporterService.java | 6 ++++++ .../src/main/resources/application.properties | 2 +- 10 files changed, 36 insertions(+), 22 deletions(-) rename clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/{DBFExportConfig.java => DBFExporterConfig.java} (84%) diff --git a/clearing-parent/dbf-exporter/pom.xml b/clearing-parent/dbf-exporter/pom.xml index 07e679fee..dc03fe6c8 100644 --- a/clearing-parent/dbf-exporter/pom.xml +++ b/clearing-parent/dbf-exporter/pom.xml @@ -53,16 +53,6 @@ - - - src/main/resources - - application.properties - - false - - - org.springframework.boot diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExportConfig.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java similarity index 84% rename from clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExportConfig.java rename to clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java index 8be4a7799..cbad9106a 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExportConfig.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java @@ -22,11 +22,11 @@ import java.util.List; @Configuration @EnableConfigurationProperties @ComponentScan(basePackages = {"ru.spcex.clearing.dbf.exporter"}) -public class DBFExportConfig { +public class DBFExporterConfig { private final AProperties properties; private final ApplicationContext context; - public DBFExportConfig(@Qualifier("dbfExporterProperties") AProperties properties, ApplicationContext context) { + public DBFExporterConfig(@Qualifier("dbfExporterProperties") AProperties properties, ApplicationContext context) { this.properties = properties; this.context = context; } @@ -60,7 +60,11 @@ public class DBFExportConfig { public ThreadPoolTaskExecutor executor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setMaxPoolSize(properties.getThreadsCount()); - executor.setThreadNamePrefix("dbf-exporter"); + executor.setCorePoolSize(properties.getThreadsCount()); + executor.setThreadNamePrefix("dbf-exporter-thread-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(300); + executor.initialize(); return executor; } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/PrepareDBFFile.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/PrepareDBFFile.java index cae1d707a..ce639423b 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/PrepareDBFFile.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/PrepareDBFFile.java @@ -12,6 +12,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Objects; /** @@ -19,6 +21,7 @@ import java.util.Objects; */ @Component public class PrepareDBFFile extends Stage implements InitializingBean { + private static DateTimeFormatter tsFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss"); private final AProperties properties; private String outDir; @@ -31,15 +34,16 @@ public class PrepareDBFFile extends Stage implements InitializingBean { Objects.requireNonNull(resultContainer.getTableForExport()); Table table = resultContainer.getTableForExport(); - File dbfFile = new File(String.format("%s%s%s.dbf", + File dbfFile = new File(String.format("%s%s%s_%s.dbf", outDir, File.separator, - table.getFilePrefix())); + table.getFilePrefix(), + tsFormatter.format(LocalDateTime.now()))); try { Path dbfFilePath = dbfFile.toPath(); Files.deleteIfExists(dbfFilePath); Files.createFile(dbfFilePath); - } catch (IOException e) { + } catch (Exception e) { log.error(String.format("uuid %s. Can't create file %s", resultContainer.getUuid(), dbfFile.getName()), e); return StageResult.ERROR; } diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/Processor.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/Processor.java index 5fc31168e..755020294 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/Processor.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/Processor.java @@ -29,7 +29,7 @@ public class Processor { if (Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result)) break; } long endMills = System.currentTimeMillis(); - log.info("uuid {}. Task completed, result: {}, time working: {}", + log.info("uuid {}. Task completed, result: {}, time working: {} ms", task.getUuid(), result, endMills - startMills); diff --git a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java index e199d31f9..5da7e1052 100644 --- a/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java +++ b/clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java @@ -8,6 +8,7 @@ import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer; import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table; import ru.spcex.clearing.dbf.exporter.logic.stages.Processor; +import javax.annotation.PreDestroy; import java.util.Map; @Service("dbfExportService") @@ -25,7 +26,12 @@ public class DBFExportService { for (Map.Entry tableForExport : tablesForExport.entrySet()) { Table table = tableForExport.getKey(); ISqlFilter filter = tableForExport.getValue(); - executor.execute(() -> processor.process(ResultContainer.createNewTask(table, filter))); + executor.submit(() -> processor.process(ResultContainer.createNewTask(table, filter))); } } + + @PreDestroy + public void shutdownTaskExecutor() { + executor.shutdown(); + } } diff --git a/clearing-parent/dbf-exporter/src/main/resources/application.properties b/clearing-parent/dbf-exporter/src/main/resources/application.properties index 2c2a05216..089043691 100644 --- a/clearing-parent/dbf-exporter/src/main/resources/application.properties +++ b/clearing-parent/dbf-exporter/src/main/resources/application.properties @@ -8,6 +8,6 @@ db.login=clearing db.password=Aa111111 dbf.encoding=cp866 -dbf.threads-count=1 +dbf.threads-count=10 dbf.out-dir=D:\\dbf\\out diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java index 1ac82b01f..d330fc660 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java @@ -62,7 +62,11 @@ public class DBFImporterConfig { public ThreadPoolTaskExecutor executor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setMaxPoolSize(properties.getThreadsCount()); - executor.setThreadNamePrefix("dbf-loader"); + executor.setCorePoolSize(properties.getThreadsCount()); + executor.setThreadNamePrefix("dbf-importer-thread-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(300); + executor.initialize(); return executor; } } diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/Processor.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/Processor.java index 3ae0aa4bc..a9320e139 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/Processor.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/Processor.java @@ -29,7 +29,7 @@ public class Processor { if (Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result)) break; } long endMills = System.currentTimeMillis(); - log.info("uuid {}. Task completed, result: {}, time working: {}", + log.info("uuid {}. Task completed, result: {}, time working: {} ms", task.getUuid(), result, endMills - startMills); diff --git a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java index b1a8e0859..99523b9d8 100644 --- a/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java +++ b/clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java @@ -9,6 +9,7 @@ import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; import ru.spcex.clearing.dbf.importer.logic.stages.Processor; +import javax.annotation.PreDestroy; import java.io.File; import java.util.List; import java.util.Map; @@ -39,4 +40,9 @@ public class DBFImporterService { } } } + + @PreDestroy + public void shutdownTaskExecutor() { + executorService.shutdown(); + } } diff --git a/clearing-parent/dbf-importer/src/main/resources/application.properties b/clearing-parent/dbf-importer/src/main/resources/application.properties index 7ba588894..451d47ee2 100644 --- a/clearing-parent/dbf-importer/src/main/resources/application.properties +++ b/clearing-parent/dbf-importer/src/main/resources/application.properties @@ -14,4 +14,4 @@ dbf.insert-batch-size=100 dbf.delete-src-files=false dbf.src-dir=D:\\dbf\\ -dbf.threads-count=1 \ No newline at end of file +dbf.threads-count=10 \ No newline at end of file