1) перемещение данных из одной директории в другую
2) выполнил заполнение полей generation_id, generation_time, file_name
This commit is contained in:
aalehin
parent
0726c3e321
commit
5ad4034388
12 changed files with 108 additions and 23 deletions
|
|
@ -7,10 +7,7 @@ import org.springframework.context.annotation.Bean;
|
|||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
import ru.spcex.clearing.dbf.importer.logic.stages.ImportToDB;
|
||||
import ru.spcex.clearing.dbf.importer.logic.stages.LoadFileFromDisk;
|
||||
import ru.spcex.clearing.dbf.importer.logic.stages.Stage;
|
||||
import ru.spcex.clearing.dbf.importer.logic.stages.ValidateFields;
|
||||
import ru.spcex.clearing.dbf.importer.logic.stages.*;
|
||||
import ru.spcex.clearing.dbf.importer.properties.AProperties;
|
||||
|
||||
import java.util.LinkedList;
|
||||
|
|
@ -36,6 +33,7 @@ public class DBFImporterConfig {
|
|||
pipeline.add(context.getBean(LoadFileFromDisk.class));
|
||||
pipeline.add(context.getBean(ValidateFields.class));
|
||||
pipeline.add(context.getBean(ImportToDB.class));
|
||||
pipeline.add(context.getBean(ChangeDirOfFileStage.class));
|
||||
|
||||
return pipeline;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
|
|||
private final String nameOfMap;
|
||||
protected HazelcastService hazelcastService;
|
||||
protected ImdgHazelcast<T> map;
|
||||
protected String filename;
|
||||
|
||||
protected AbstractTable(String prefix, Class<T> clazz, String nameOfMap) {
|
||||
this.prefix = prefix;
|
||||
|
|
@ -17,6 +18,10 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
|
|||
this.nameOfMap = nameOfMap;
|
||||
}
|
||||
|
||||
public void setFilename(String filename) {
|
||||
this.filename = filename;
|
||||
}
|
||||
|
||||
public void setHazelcastService(HazelcastService hazelcastService) {
|
||||
this.hazelcastService = hazelcastService;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import ru.clearing.classes.statics.data.sdf.SDf01;
|
|||
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
|
||||
import ru.spcex.clearing.imdg.IMDGDistributedNames;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
public class SDf01Table extends AbstractTable<SDf01> {
|
||||
|
||||
private static final String PREFIX = ETable.DF_01.name();
|
||||
|
|
@ -30,6 +32,9 @@ public class SDf01Table extends AbstractTable<SDf01> {
|
|||
result.setSumengage((String) entity[9]);
|
||||
result.setSumunblock((String) entity[10]);
|
||||
result.setFile_type((String) entity[11]);
|
||||
result.setFileName(filename);
|
||||
result.setGenerationTime(Instant.now());
|
||||
result.setGenerationId(0L);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
|
|||
import ru.spcex.clearing.imdg.IMDGDistributedNames;
|
||||
import ru.spcex.platform.utils.time.TimeUtil;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
public class SDf02Table extends AbstractTable<SDf02> {
|
||||
|
||||
private static final String PREFIX = ETable.DF_02.name();
|
||||
|
|
@ -35,6 +37,9 @@ public class SDf02Table extends AbstractTable<SDf02> {
|
|||
result.setGenerationTime(TimeUtil.strToInstant((String) entity[13]));
|
||||
result.setGenerationId((Long) entity[14]);
|
||||
result.setInSDf01Id((Long) entity[15]);
|
||||
//result.setFileName(filename);
|
||||
result.setGenerationTime(Instant.now());
|
||||
result.setGenerationId(0L);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import ru.spcex.clearing.imdg.IMDGDistributedNames;
|
|||
import ru.spcex.platform.utils.time.TimeUtil;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.Instant;
|
||||
|
||||
public class SDf08Table extends AbstractTable<SDf08> {
|
||||
|
||||
|
|
@ -24,8 +25,9 @@ public class SDf08Table extends AbstractTable<SDf08> {
|
|||
|
||||
result.setNumber((BigDecimal) entity[0]);
|
||||
result.setDatetime(TimeUtil.strToInstant((String) entity[1]));
|
||||
result.setGenerationTime(TimeUtil.strToInstant((String) entity[2]));
|
||||
result.setGenerationId((Long) entity[3]);
|
||||
//result.setFileName(filename);
|
||||
result.setGenerationTime(Instant.now());
|
||||
result.setGenerationId(0L);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables;
|
|||
import ru.clearing.classes.statics.data.sdf.SDf12;
|
||||
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
|
||||
import ru.spcex.clearing.imdg.IMDGDistributedNames;
|
||||
import ru.spcex.platform.utils.time.TimeUtil;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.Instant;
|
||||
|
||||
public class SDf12Table extends AbstractTable<SDf12> {
|
||||
|
||||
|
|
@ -25,10 +25,9 @@ public class SDf12Table extends AbstractTable<SDf12> {
|
|||
result.setAccount((String) entity[0]);
|
||||
result.setDeal((String) entity[1]);
|
||||
result.setStatus((BigDecimal) entity[2]);
|
||||
result.setFileName((String) entity[3]);
|
||||
result.setGenerationTime(TimeUtil.strToInstant((String) entity[4]));
|
||||
result.setGenerationId((Long) entity[5]);
|
||||
|
||||
result.setFileName(filename);
|
||||
result.setGenerationTime(Instant.now());
|
||||
result.setGenerationId(0L);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import ru.spcex.clearing.imdg.IMDGDistributedNames;
|
|||
import ru.spcex.platform.utils.time.TimeUtil;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.Instant;
|
||||
|
||||
public class SDf16Table extends AbstractTable<SDf16> {
|
||||
|
||||
|
|
@ -31,9 +32,9 @@ public class SDf16Table extends AbstractTable<SDf16> {
|
|||
result.setBIC((BigDecimal) entity[6]);
|
||||
result.setSPEC((String) entity[7]);
|
||||
result.setNumber((BigDecimal) entity[8]);
|
||||
result.setFileName((String) entity[9]);
|
||||
result.setGenerationTime(TimeUtil.strToInstant((String) entity[10]));
|
||||
result.setGenerationId((Long) entity[11]);
|
||||
result.setFileName(filename);
|
||||
result.setGenerationTime(Instant.now());
|
||||
result.setGenerationId(0L);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -3,9 +3,9 @@ package ru.spcex.clearing.dbf.importer.logic.data.tables;
|
|||
import ru.clearing.classes.statics.data.sdf.SDf18;
|
||||
import ru.spcex.clearing.dbf.importer.logic.data.enums.ETable;
|
||||
import ru.spcex.clearing.imdg.IMDGDistributedNames;
|
||||
import ru.spcex.platform.utils.time.TimeUtil;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.Instant;
|
||||
|
||||
public class SDf18Table extends AbstractTable<SDf18> {
|
||||
|
||||
|
|
@ -20,13 +20,14 @@ public class SDf18Table extends AbstractTable<SDf18> {
|
|||
@Override
|
||||
public SDf18 getEntity(Object[] entity) {
|
||||
SDf18 result = new SDf18();
|
||||
|
||||
|
||||
result.setAccount((String) entity[0]);
|
||||
result.setDeal((String) entity[1]);
|
||||
result.setStatus((BigDecimal) entity[2]);
|
||||
result.setResult((BigDecimal) entity[3]);
|
||||
result.setGenerationTime(TimeUtil.strToInstant((String) entity[4]));
|
||||
result.setGenerationId((Long) entity[5]);
|
||||
// result.setFileName(filename);
|
||||
result.setGenerationTime(Instant.now());
|
||||
result.setGenerationId(0L);
|
||||
result.setInSDf12Id((Long) entity[6]);
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
package ru.spcex.clearing.dbf.importer.logic.stages;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
|
||||
import ru.spcex.clearing.dbf.importer.logic.data.enums.StageResult;
|
||||
import ru.spcex.clearing.dbf.importer.properties.AProperties;
|
||||
import ru.spcex.platform.utils.time.TimeUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Instant;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
|
||||
|
||||
|
||||
@Component
|
||||
public class ChangeDirOfFileStage extends Stage {
|
||||
|
||||
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy.MM.dd HH.mm.ss");
|
||||
private final AProperties properties;
|
||||
|
||||
@Autowired
|
||||
public ChangeDirOfFileStage(AProperties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StageResult process(ResultContainer resultContainer) {
|
||||
File srcDir = new File(properties.getSrcDir());
|
||||
File outDir = new File(properties.getOutDir());
|
||||
File dbfFile = resultContainer.getDbfFile();
|
||||
|
||||
if (!srcDir.exists()) {
|
||||
log.error("SettlementHouse_DocIn does not exists!");
|
||||
return StageResult.ERROR;
|
||||
}
|
||||
if (!outDir.exists()) {
|
||||
log.warn("SettlementHouse_DocIn does not exists! Trying to made new!");
|
||||
if (outDir.mkdir()) {
|
||||
log.error("Made dir {} successfully!", outDir.getName());
|
||||
} else {
|
||||
log.error("SettlementHouse_DocIn does not exists!");
|
||||
return StageResult.ERROR;
|
||||
}
|
||||
}
|
||||
if (!dbfFile.exists()) {
|
||||
log.error("DBF file to save does not exists!");
|
||||
return StageResult.ERROR;
|
||||
}
|
||||
String newNameOfFile = TimeUtil.formatInstant(Instant.now(), FORMATTER) + '_' + dbfFile.getName();
|
||||
|
||||
try {
|
||||
Files.move(dbfFile.toPath(), outDir.toPath().resolve(newNameOfFile), REPLACE_EXISTING);
|
||||
} catch (IOException e) {
|
||||
log.error("Could not move file {}", e.getMessage());
|
||||
return StageResult.ERROR;
|
||||
}
|
||||
|
||||
return StageResult.COMPLETE;
|
||||
}
|
||||
}
|
||||
|
|
@ -43,6 +43,7 @@ public class ImportToDB extends Stage {
|
|||
DBFReader dbfReader = new DBFReader(is, sourceCharset)) {
|
||||
AbstractTable table = MAPPING_ENUM_TABLE_OBJECT_TABLE.get(currTable);
|
||||
table.setHazelcastService(hazelcastService);
|
||||
table.setFilename(resultContainer.getDbfFile().getName());
|
||||
for (int i = 0; i < dbfReader.getRecordCount(); i++) {
|
||||
Object[] entity = dbfReader.nextRecord();
|
||||
table.injectEntity(table.getEntity(entity));
|
||||
|
|
@ -53,6 +54,6 @@ public class ImportToDB extends Stage {
|
|||
}
|
||||
|
||||
|
||||
return StageResult.COMPLETE;
|
||||
return StageResult.OK;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,19 +8,21 @@ public class AProperties {
|
|||
|
||||
@Value("${dbf.src-dir}")
|
||||
private String srcDir;
|
||||
|
||||
@Value("${dbf.out-dir}")
|
||||
private String outDir;
|
||||
@Value("${dbf.delete-src-files}")
|
||||
private boolean deleteSrcFiles = true;
|
||||
|
||||
@Value("${dbf.encoding-source}")
|
||||
private String dbfEncoding;
|
||||
|
||||
@Value("${dbf.insert-batch-size}")
|
||||
private int insertBatchSize;
|
||||
|
||||
@Value("${dbf.threads-count}")
|
||||
private int threadsCount;
|
||||
|
||||
public String getOutDir() {
|
||||
return outDir;
|
||||
}
|
||||
|
||||
public String getSrcDir() {
|
||||
return srcDir;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ dbf.encoding-source=cp866
|
|||
dbf.insert-batch-size=100
|
||||
dbf.delete-src-files=false
|
||||
dbf.src-dir=D:\\dbf\\
|
||||
dbf.out-dir=D:\\dbf\\out
|
||||
dbf.threads-count=10
|
||||
import-dbf-service.hazelcast.cluster-members=127.0.0.1
|
||||
import-dbf-service.hazelcast.login=dev
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue