---
improvement
This commit is contained in:
aalehin 2023-05-27 16:09:03 +03:00
parent 9b8c2d3e37
commit eafd571518
11 changed files with 85 additions and 47 deletions

View file

@ -20,10 +20,10 @@ public class ResultContainer {
protected ResultContainer() { protected ResultContainer() {
} }
public static ResultContainer createNewTask(ETable dbfTable, File dbfFile) { public static ResultContainer createNewTask(ETable swtTable, File swtFile) {
ResultContainer container = new ResultContainer(); ResultContainer container = new ResultContainer();
container.swtTable = dbfTable; container.swtTable = swtTable;
container.swtFile = dbfFile; container.swtFile = swtFile;
container.uuid = UUID.randomUUID(); container.uuid = UUID.randomUUID();
return container; return container;
} }

View file

@ -1,10 +1,10 @@
package ru.spcex.clearing.swt.importer.logic.data.enums; package ru.spcex.clearing.swt.importer.logic.data.enums;
public enum ETable { public enum ETable {
S_DF_08("S_DF08"), S_DF_08("DF-08"),
S_DF_10("S_DF10"), S_DF_10("DF-10"),
S_DF_13("S_DF13"); S_DF_13("DF-13");
//S_DF20("S_DF20"); //S_DF20("DF-20");
private final String prefix; private final String prefix;
@ -28,6 +28,6 @@ public enum ETable {
} }
public boolean fileForThisTable(String filename) { public boolean fileForThisTable(String filename) {
return filename != null && filename.startsWith(prefix); return filename != null && filename.contains(prefix);
}
} }
}

View file

@ -31,7 +31,7 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
this.hazelcastService = hazelcastService; this.hazelcastService = hazelcastService;
} }
public abstract T getEntity(Object[] entity); public abstract T getEntity(String[] entity);
public void injectEntity(T obj) { public void injectEntity(T obj) {
if (map == null) { if (map == null) {

View file

@ -17,13 +17,13 @@ public class SDf08Table extends AbstractTable<SDf08> {
} }
@Override @Override
public SDf08 getEntity(Object[] entity) { public SDf08 getEntity(String[] entity) {
SDf08 result = new SDf08(); SDf08 result = new SDf08();
result.setOutDocument((String) entity[0]); result.setOutDocument(entity[3]);
result.setDepoCode((String) entity[1]); result.setDepoCode(entity[0]);
result.setQuantity((String) entity[2]); result.setQuantity(entity[1]);
result.setSecurityCode((String) entity[3]); result.setSecurityCode(entity[2]);
result.setClientName((String) entity[4]); result.setClientName(entity[5]);
result.setFileName(filename); result.setFileName(filename);
result.setGenerationTime(Instant.now()); result.setGenerationTime(Instant.now());
result.setGenerationId(fileId); result.setGenerationId(fileId);

View file

@ -17,13 +17,13 @@ public class SDf10Table extends AbstractTable<SDf10> {
} }
@Override @Override
public SDf10 getEntity(Object[] entity) { public SDf10 getEntity(String[] entity) {
SDf10 result = new SDf10(); SDf10 result = new SDf10();
result.setOutDocument((String) entity[0]); result.setOutDocument(entity[3]);
result.setDepoCode((String) entity[1]); result.setDepoCode(entity[0]);
result.setQuantity((String) entity[2]); result.setQuantity(entity[1]);
result.setSecurityCode((String) entity[3]); result.setSecurityCode(entity[2]);
result.setClientName((String) entity[4]); result.setClientName(entity[5]);
result.setFileName(filename); result.setFileName(filename);
result.setGenerationTime(Instant.now()); result.setGenerationTime(Instant.now());
result.setGenerationId(fileId); result.setGenerationId(fileId);

View file

@ -17,9 +17,8 @@ public class SDf13Table extends AbstractTable<SDf13> {
} }
@Override @Override
public SDf13 getEntity(Object[] entity) { public SDf13 getEntity(String[] entity) {
SDf13 result = new SDf13(); SDf13 result = new SDf13();
result.setFileName(filename); result.setFileName(filename);
result.setGenerationTime(Instant.now()); result.setGenerationTime(Instant.now());
result.setGenerationId(fileId); result.setGenerationId(fileId);

View file

@ -1,6 +1,5 @@
package ru.spcex.clearing.swt.importer.logic.stages; package ru.spcex.clearing.swt.importer.logic.stages;
import com.linuxense.javadbf.DBFReader;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings; import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
@ -8,6 +7,7 @@ import ru.spcex.clearing.swt.importer.logic.data.ResultContainer;
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable; import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult; import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable; import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
import ru.spcex.clearing.swt.importer.util.SWTReader;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
import ru.spcex.platform.utils.log.ExceptionUtils; import ru.spcex.platform.utils.log.ExceptionUtils;
@ -43,17 +43,21 @@ public class ImportToDB extends Stage {
byte[] source = resultContainer.getSwtSource(); byte[] source = resultContainer.getSwtSource();
Charset sourceCharset = Charset.forName(settings.getCommon().getEncodingSource()); Charset sourceCharset = Charset.forName(settings.getCommon().getEncodingSource());
try (InputStream is = new ByteArrayInputStream(source); try (InputStream is = new ByteArrayInputStream(source);
DBFReader dbfReader = new DBFReader(is, sourceCharset)) { SWTReader swtReader = new SWTReader(is, sourceCharset)) {
AbstractTable table = mappingEnumTableObjectTable.get(currTable); AbstractTable table = mappingEnumTableObjectTable.get(currTable);
table.setHazelcastService(hazelcastService); table.setHazelcastService(hazelcastService);
table.setFilename(resultContainer.getSwtFile().getName()); table.setFilename(resultContainer.getSwtFile().getName());
Long fileId = hazelcastService.getImdgIdGenerator().nextId(); Long fileId = hazelcastService.getImdgIdGenerator().nextId();
table.setFileId(fileId); table.setFileId(fileId);
for (int i = 0; i < dbfReader.getRecordCount(); i++) { int counter = 0;
Object[] entity = dbfReader.nextRecord(); while (swtReader.hasNextRecord()) {
counter++;
String[] entity = swtReader.nextRecord();
if (entity == null) { if (entity == null) {
log.warn("record index {} null", i); log.warn("record index {} null", counter);
continue; continue;
} }
table.injectEntity(table.getEntity(entity)); table.injectEntity(table.getEntity(entity));

View file

@ -25,13 +25,13 @@ public class LoadFileFromDisk extends Stage {
Objects.requireNonNull(resultContainer.getSwtFile()); Objects.requireNonNull(resultContainer.getSwtFile());
UUID taskUuid = resultContainer.getUuid(); UUID taskUuid = resultContainer.getUuid();
File dbfFile = resultContainer.getSwtFile(); File swtFile = resultContainer.getSwtFile();
byte[] fileBytes; byte[] fileBytes;
try { try {
log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getSwtFile().getName()); log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getSwtFile().getName());
fileBytes = Files.readAllBytes(Paths.get(dbfFile.getAbsolutePath())); fileBytes = Files.readAllBytes(Paths.get(swtFile.getAbsolutePath()));
if (fileBytes.length == 0) throw new IOException("Empty file"); if (fileBytes.length == 0) throw new IOException("Empty file");
if (settings.getStore().isDeleteSrcFiles()) Files.delete(dbfFile.toPath()); if (settings.getStore().isDeleteSrcFiles()) Files.delete(swtFile.toPath());
} catch (IOException e) { } catch (IOException e) {
log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getSwtFile().getName()), e); log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getSwtFile().getName()), e);
return StageResult.ERROR; return StageResult.ERROR;

View file

@ -27,7 +27,7 @@ public class ValidateFields extends Stage implements InitializingBean {
} }
/** /**
* 1. Проверяет кодировку из настройки dbf.encoding-source * 1. Проверяет кодировку из настройки swt.encoding-source
*/ */
@Override @Override
public void afterPropertiesSet() throws Exception { public void afterPropertiesSet() throws Exception {

View file

@ -23,32 +23,31 @@ public class FileChecker {
Map<ETable, List<File>> newFiles = new EnumMap<>(ETable.class); Map<ETable, List<File>> newFiles = new EnumMap<>(ETable.class);
String srcDir = settings.getStore().getSrcDir(); String srcDir = settings.getStore().getSrcDir();
List<File> dbfFiles = lsDBF(srcDir); List<File> swtFiles = lsSWT(srcDir);
if (dbfFiles.isEmpty()) return newFiles; if (swtFiles.isEmpty()) return newFiles;
for (File dbfFile : dbfFiles) { for (File swtFile : swtFiles) {
if (dbfFile.isDirectory()) continue; if (swtFile.isDirectory()) continue;
ETable currTable = ETable.getTableForFilename(dbfFile.getName()); ETable currTable = ETable.getTableForFilename(swtFile.getName());
if (currTable == null) continue; if (currTable == null) continue;
if (specificTable != null && !specificTable.equals(currTable)) continue; if (specificTable != null && !specificTable.equals(currTable)) continue;
List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>()); List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());
currList.add(dbfFile); currList.add(swtFile);
} }
return newFiles; return newFiles;
} }
private List<File> lsDBF(String dbfDirPath) { private List<File> lsSWT(String swtDirPath) {
File dbfDir = new File(dbfDirPath); File swtDir = new File(swtDirPath);
File[] dbfFiles = dbfDir.listFiles((dir, name) -> { File[] swtFiles = swtDir.listFiles((dir, name) -> {
int formatPosition = name.lastIndexOf("."); String offsetName = name.substring(0, 9);
if (formatPosition == -1 || formatPosition == name.length() - 1) return false; return offsetName.equalsIgnoreCase("RDC_KS_DF");
return "dbf".equalsIgnoreCase(name.substring(formatPosition + 1));
}); });
List<File> resultFiles = new LinkedList<>(); List<File> resultFiles = new LinkedList<>();
if (dbfFiles != null && dbfFiles.length >= 1) { if (swtFiles != null && swtFiles.length >= 1) {
resultFiles.addAll(Arrays.asList(dbfFiles)); resultFiles.addAll(Arrays.asList(swtFiles));
} }
return resultFiles; return resultFiles;

View file

@ -0,0 +1,36 @@
package ru.spcex.clearing.swt.importer.util;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Scanner;
public class SWTReader implements Closeable {
protected InputStream inputStream;
protected Scanner scanner;
public SWTReader(InputStream inputStream, Charset charset) {
this.inputStream = inputStream;
this.scanner = new Scanner(inputStream, charset);
}
public String[] nextRecord() {
String line = scanner.nextLine();
String[] separetedLine = line.split(":");
if (separetedLine.length < 5) {
return null;
}
return separetedLine;
}
public boolean hasNextRecord() {
return scanner.hasNextLine();
}
@Override
public void close() throws IOException {
inputStream.close();
scanner.close();
}
}