treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dbf exporter and fix dbf importer

Browse source
This commit is contained in:
akulikov 2022-08-11 19:46:48 +03:00
parent bbd7fb2989
commit 7009933409
10 changed files with 36 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
clearing-parent/dbf-exporter/pom.xml
View file

@ -53,16 +53,6 @@
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>application.properties</exclude>
</excludes>
<filtering>false</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>

10
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExportConfig.java → clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/config/DBFExporterConfig.java
View file

@ -22,11 +22,11 @@ import java.util.List;
@Configuration
@EnableConfigurationProperties
@ComponentScan(basePackages = {"ru.spcex.clearing.dbf.exporter"})
public class DBFExportConfig {
public class DBFExporterConfig {
private final AProperties properties;
private final ApplicationContext context;
public DBFExportConfig(@Qualifier("dbfExporterProperties") AProperties properties, ApplicationContext context) {
public DBFExporterConfig(@Qualifier("dbfExporterProperties") AProperties properties, ApplicationContext context) {
this.properties = properties;
this.context = context;
}
@ -60,7 +60,11 @@ public class DBFExportConfig {
public ThreadPoolTaskExecutor executor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setMaxPoolSize(properties.getThreadsCount());
executor.setThreadNamePrefix("dbf-exporter");
executor.setCorePoolSize(properties.getThreadsCount());
executor.setThreadNamePrefix("dbf-exporter-thread-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(300);
executor.initialize();
return executor;
}

10
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/PrepareDBFFile.java
View file

@ -12,6 +12,8 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
/**
@ -19,6 +21,7 @@ import java.util.Objects;
*/
@Component
public class PrepareDBFFile extends Stage implements InitializingBean {
private static DateTimeFormatter tsFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss");
private final AProperties properties;
private String outDir;
@ -31,15 +34,16 @@ public class PrepareDBFFile extends Stage implements InitializingBean {
Objects.requireNonNull(resultContainer.getTableForExport());
Table table = resultContainer.getTableForExport();
File dbfFile = new File(String.format("%s%s%s.dbf",
File dbfFile = new File(String.format("%s%s%s_%s.dbf",
outDir,
File.separator,
table.getFilePrefix()));
table.getFilePrefix(),
tsFormatter.format(LocalDateTime.now())));
try {
Path dbfFilePath = dbfFile.toPath();
Files.deleteIfExists(dbfFilePath);
Files.createFile(dbfFilePath);
} catch (IOException e) {
} catch (Exception e) {
log.error(String.format("uuid %s. Can't create file %s", resultContainer.getUuid(), dbfFile.getName()), e);
return StageResult.ERROR;
}

2
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/logic/stages/Processor.java
View file

@ -29,7 +29,7 @@ public class Processor {
if (Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result)) break;
}
long endMills = System.currentTimeMillis();
log.info("uuid {}. Task completed, result: {}, time working: {}",
log.info("uuid {}. Task completed, result: {}, time working: {} ms",
task.getUuid(),
result,
endMills - startMills);

8
clearing-parent/dbf-exporter/src/main/java/ru/spcex/clearing/dbf/exporter/services/DBFExportService.java
View file

@ -8,6 +8,7 @@ import ru.spcex.clearing.dbf.exporter.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.exporter.logic.data.enums.Table;
import ru.spcex.clearing.dbf.exporter.logic.stages.Processor;
import javax.annotation.PreDestroy;
import java.util.Map;
@Service("dbfExportService")
@ -25,7 +26,12 @@ public class DBFExportService {
for (Map.Entry<Table, ISqlFilter> tableForExport : tablesForExport.entrySet()) {
Table table = tableForExport.getKey();
ISqlFilter filter = tableForExport.getValue();
executor.execute(() -> processor.process(ResultContainer.createNewTask(table, filter)));
executor.submit(() -> processor.process(ResultContainer.createNewTask(table, filter)));
}
}
@PreDestroy
public void shutdownTaskExecutor() {
executor.shutdown();
}
}

2
clearing-parent/dbf-exporter/src/main/resources/application.properties
View file

@ -8,6 +8,6 @@ db.login=clearing
db.password=Aa111111
dbf.encoding=cp866
dbf.threads-count=1
dbf.threads-count=10
dbf.out-dir=D:\\dbf\\out

6
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/config/DBFImporterConfig.java
View file

@ -62,7 +62,11 @@ public class DBFImporterConfig {
public ThreadPoolTaskExecutor executor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setMaxPoolSize(properties.getThreadsCount());
executor.setThreadNamePrefix("dbf-loader");
executor.setCorePoolSize(properties.getThreadsCount());
executor.setThreadNamePrefix("dbf-importer-thread-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(300);
executor.initialize();
return executor;
}
}

2
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/Processor.java
View file

@ -29,7 +29,7 @@ public class Processor {
if (Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result)) break;
}
long endMills = System.currentTimeMillis();
log.info("uuid {}. Task completed, result: {}, time working: {}",
log.info("uuid {}. Task completed, result: {}, time working: {} ms",
task.getUuid(),
result,
endMills - startMills);

6
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java
View file

@ -9,6 +9,7 @@ import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.stages.Processor;
import javax.annotation.PreDestroy;
import java.io.File;
import java.util.List;
import java.util.Map;
@ -39,4 +40,9 @@ public class DBFImporterService {
}
}
}
@PreDestroy
public void shutdownTaskExecutor() {
executorService.shutdown();
}
}

2
clearing-parent/dbf-importer/src/main/resources/application.properties
View file

@ -14,4 +14,4 @@ dbf.insert-batch-size=100
dbf.delete-src-files=false
dbf.src-dir=D:\\dbf\\
dbf.threads-count=1
dbf.threads-count=10