treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

http://jira.mfd.msk:8088/browse/CLS-316

Browse source 
---
improvement
This commit is contained in:
aalehin 2023-05-27 16:09:03 +03:00
parent 9b8c2d3e37
commit eafd571518
11 changed files with 85 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/ResultContainer.java
View file

@ -20,10 +20,10 @@ public class ResultContainer {
protected ResultContainer() {
}
public static ResultContainer createNewTask(ETable dbfTable, File dbfFile) {
public static ResultContainer createNewTask(ETable swtTable, File swtFile) {
ResultContainer container = new ResultContainer();
container.swtTable = dbfTable;
container.swtFile = dbfFile;
container.swtTable = swtTable;
container.swtFile = swtFile;
container.uuid = UUID.randomUUID();
return container;
}

12
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/enums/ETable.java
View file

@ -1,10 +1,10 @@
package ru.spcex.clearing.swt.importer.logic.data.enums;
public enum ETable {
S_DF_08("S_DF08"),
S_DF_10("S_DF10"),
S_DF_13("S_DF13");
//S_DF20("S_DF20");
S_DF_08("DF-08"),
S_DF_10("DF-10"),
S_DF_13("DF-13");
//S_DF20("DF-20");
private final String prefix;
@ -28,6 +28,6 @@ public enum ETable {
}
public boolean fileForThisTable(String filename) {
return filename != null && filename.startsWith(prefix);
}
return filename != null && filename.contains(prefix);
}
}

2
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/AbstractTable.java
View file

@ -31,7 +31,7 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
this.hazelcastService = hazelcastService;
}
public abstract T getEntity(Object[] entity);
public abstract T getEntity(String[] entity);
public void injectEntity(T obj) {
if (map == null) {

12
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf08Table.java
View file

@ -17,13 +17,13 @@ public class SDf08Table extends AbstractTable<SDf08> {
}
@Override
public SDf08 getEntity(Object[] entity) {
public SDf08 getEntity(String[] entity) {
SDf08 result = new SDf08();
result.setOutDocument((String) entity[0]);
result.setDepoCode((String) entity[1]);
result.setQuantity((String) entity[2]);
result.setSecurityCode((String) entity[3]);
result.setClientName((String) entity[4]);
result.setOutDocument(entity[3]);
result.setDepoCode(entity[0]);
result.setQuantity(entity[1]);
result.setSecurityCode(entity[2]);
result.setClientName(entity[5]);
result.setFileName(filename);
result.setGenerationTime(Instant.now());
result.setGenerationId(fileId);

12
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf10Table.java
View file

@ -17,13 +17,13 @@ public class SDf10Table extends AbstractTable<SDf10> {
}
@Override
public SDf10 getEntity(Object[] entity) {
public SDf10 getEntity(String[] entity) {
SDf10 result = new SDf10();
result.setOutDocument((String) entity[0]);
result.setDepoCode((String) entity[1]);
result.setQuantity((String) entity[2]);
result.setSecurityCode((String) entity[3]);
result.setClientName((String) entity[4]);
result.setOutDocument(entity[3]);
result.setDepoCode(entity[0]);
result.setQuantity(entity[1]);
result.setSecurityCode(entity[2]);
result.setClientName(entity[5]);
result.setFileName(filename);
result.setGenerationTime(Instant.now());
result.setGenerationId(fileId);

3
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/data/tables/SDf13Table.java
View file

@ -17,9 +17,8 @@ public class SDf13Table extends AbstractTable<SDf13> {
}
@Override
public SDf13 getEntity(Object[] entity) {
public SDf13 getEntity(String[] entity) {
SDf13 result = new SDf13();
result.setFileName(filename);
result.setGenerationTime(Instant.now());
result.setGenerationId(fileId);

14
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ImportToDB.java
View file

@ -1,6 +1,5 @@
package ru.spcex.clearing.swt.importer.logic.stages;
import com.linuxense.javadbf.DBFReader;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
@ -8,6 +7,7 @@ import ru.spcex.clearing.swt.importer.logic.data.ResultContainer;
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
import ru.spcex.clearing.swt.importer.util.SWTReader;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import ru.spcex.platform.utils.log.ExceptionUtils;
@ -43,17 +43,21 @@ public class ImportToDB extends Stage {
byte[] source = resultContainer.getSwtSource();
Charset sourceCharset = Charset.forName(settings.getCommon().getEncodingSource());
try (InputStream is = new ByteArrayInputStream(source);
DBFReader dbfReader = new DBFReader(is, sourceCharset)) {
SWTReader swtReader = new SWTReader(is, sourceCharset)) {
AbstractTable table = mappingEnumTableObjectTable.get(currTable);
table.setHazelcastService(hazelcastService);
table.setFilename(resultContainer.getSwtFile().getName());
Long fileId = hazelcastService.getImdgIdGenerator().nextId();
table.setFileId(fileId);
for (int i = 0; i < dbfReader.getRecordCount(); i++) {
Object[] entity = dbfReader.nextRecord();
int counter = 0;
while (swtReader.hasNextRecord()) {
counter++;
String[] entity = swtReader.nextRecord();
if (entity == null) {
log.warn("record index {} null", i);
log.warn("record index {} null", counter);
continue;
}
table.injectEntity(table.getEntity(entity));

6
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/LoadFileFromDisk.java
View file

@ -25,13 +25,13 @@ public class LoadFileFromDisk extends Stage {
Objects.requireNonNull(resultContainer.getSwtFile());
UUID taskUuid = resultContainer.getUuid();
File dbfFile = resultContainer.getSwtFile();
File swtFile = resultContainer.getSwtFile();
byte[] fileBytes;
try {
log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getSwtFile().getName());
fileBytes = Files.readAllBytes(Paths.get(dbfFile.getAbsolutePath()));
fileBytes = Files.readAllBytes(Paths.get(swtFile.getAbsolutePath()));
if (fileBytes.length == 0) throw new IOException("Empty file");
if (settings.getStore().isDeleteSrcFiles()) Files.delete(dbfFile.toPath());
if (settings.getStore().isDeleteSrcFiles()) Files.delete(swtFile.toPath());
} catch (IOException e) {
log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getSwtFile().getName()), e);
return StageResult.ERROR;

2
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/logic/stages/ValidateFields.java
View file

@ -27,7 +27,7 @@ public class ValidateFields extends Stage implements InitializingBean {
}
/**
* 1. Проверяет кодировку из настройки dbf.encoding-source
* 1. Проверяет кодировку из настройки swt.encoding-source
*/
@Override
public void afterPropertiesSet() throws Exception {

27
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/services/FileChecker.java
View file

@ -23,32 +23,31 @@ public class FileChecker {
Map<ETable, List<File>> newFiles = new EnumMap<>(ETable.class);
String srcDir = settings.getStore().getSrcDir();
List<File> dbfFiles = lsDBF(srcDir);
if (dbfFiles.isEmpty()) return newFiles;
List<File> swtFiles = lsSWT(srcDir);
if (swtFiles.isEmpty()) return newFiles;
for (File dbfFile : dbfFiles) {
if (dbfFile.isDirectory()) continue;
for (File swtFile : swtFiles) {
if (swtFile.isDirectory()) continue;
ETable currTable = ETable.getTableForFilename(dbfFile.getName());
ETable currTable = ETable.getTableForFilename(swtFile.getName());
if (currTable == null) continue;
if (specificTable != null && !specificTable.equals(currTable)) continue;
List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());
currList.add(dbfFile);
currList.add(swtFile);
}
return newFiles;
}
private List<File> lsDBF(String dbfDirPath) {
File dbfDir = new File(dbfDirPath);
File[] dbfFiles = dbfDir.listFiles((dir, name) -> {
int formatPosition = name.lastIndexOf(".");
if (formatPosition == -1 || formatPosition == name.length() - 1) return false;
return "dbf".equalsIgnoreCase(name.substring(formatPosition + 1));
private List<File> lsSWT(String swtDirPath) {
File swtDir = new File(swtDirPath);
File[] swtFiles = swtDir.listFiles((dir, name) -> {
String offsetName = name.substring(0, 9);
return offsetName.equalsIgnoreCase("RDC_KS_DF");
});
List<File> resultFiles = new LinkedList<>();
if (dbfFiles != null && dbfFiles.length >= 1) {
resultFiles.addAll(Arrays.asList(dbfFiles));
if (swtFiles != null && swtFiles.length >= 1) {
resultFiles.addAll(Arrays.asList(swtFiles));
}
return resultFiles;

36
clearing-parent/swt-importer/src/main/java/ru/spcex/clearing/swt/importer/util/SWTReader.java Normal file
View file

@ -0,0 +1,36 @@
package ru.spcex.clearing.swt.importer.util;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Scanner;
public class SWTReader implements Closeable {
protected InputStream inputStream;
protected Scanner scanner;
public SWTReader(InputStream inputStream, Charset charset) {
this.inputStream = inputStream;
this.scanner = new Scanner(inputStream, charset);
}
public String[] nextRecord() {
String line = scanner.nextLine();
String[] separetedLine = line.split(":");
if (separetedLine.length < 5) {
return null;
}
return separetedLine;
}
public boolean hasNextRecord() {
return scanner.hasNextLine();
}
@Override
public void close() throws IOException {
inputStream.close();
scanner.close();
}
}