parent
9b8c2d3e37
commit
eafd571518
11 changed files with 85 additions and 47 deletions
|
|
@ -20,10 +20,10 @@ public class ResultContainer {
|
||||||
protected ResultContainer() {
|
protected ResultContainer() {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ResultContainer createNewTask(ETable dbfTable, File dbfFile) {
|
public static ResultContainer createNewTask(ETable swtTable, File swtFile) {
|
||||||
ResultContainer container = new ResultContainer();
|
ResultContainer container = new ResultContainer();
|
||||||
container.swtTable = dbfTable;
|
container.swtTable = swtTable;
|
||||||
container.swtFile = dbfFile;
|
container.swtFile = swtFile;
|
||||||
container.uuid = UUID.randomUUID();
|
container.uuid = UUID.randomUUID();
|
||||||
return container;
|
return container;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
package ru.spcex.clearing.swt.importer.logic.data.enums;
|
package ru.spcex.clearing.swt.importer.logic.data.enums;
|
||||||
|
|
||||||
public enum ETable {
|
public enum ETable {
|
||||||
S_DF_08("S_DF08"),
|
S_DF_08("DF-08"),
|
||||||
S_DF_10("S_DF10"),
|
S_DF_10("DF-10"),
|
||||||
S_DF_13("S_DF13");
|
S_DF_13("DF-13");
|
||||||
//S_DF20("S_DF20");
|
//S_DF20("DF-20");
|
||||||
|
|
||||||
private final String prefix;
|
private final String prefix;
|
||||||
|
|
||||||
|
|
@ -28,6 +28,6 @@ public enum ETable {
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean fileForThisTable(String filename) {
|
public boolean fileForThisTable(String filename) {
|
||||||
return filename != null && filename.startsWith(prefix);
|
return filename != null && filename.contains(prefix);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ public abstract class AbstractTable<T extends SpcexObjectBase> {
|
||||||
this.hazelcastService = hazelcastService;
|
this.hazelcastService = hazelcastService;
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract T getEntity(Object[] entity);
|
public abstract T getEntity(String[] entity);
|
||||||
|
|
||||||
public void injectEntity(T obj) {
|
public void injectEntity(T obj) {
|
||||||
if (map == null) {
|
if (map == null) {
|
||||||
|
|
|
||||||
|
|
@ -17,13 +17,13 @@ public class SDf08Table extends AbstractTable<SDf08> {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SDf08 getEntity(Object[] entity) {
|
public SDf08 getEntity(String[] entity) {
|
||||||
SDf08 result = new SDf08();
|
SDf08 result = new SDf08();
|
||||||
result.setOutDocument((String) entity[0]);
|
result.setOutDocument(entity[3]);
|
||||||
result.setDepoCode((String) entity[1]);
|
result.setDepoCode(entity[0]);
|
||||||
result.setQuantity((String) entity[2]);
|
result.setQuantity(entity[1]);
|
||||||
result.setSecurityCode((String) entity[3]);
|
result.setSecurityCode(entity[2]);
|
||||||
result.setClientName((String) entity[4]);
|
result.setClientName(entity[5]);
|
||||||
result.setFileName(filename);
|
result.setFileName(filename);
|
||||||
result.setGenerationTime(Instant.now());
|
result.setGenerationTime(Instant.now());
|
||||||
result.setGenerationId(fileId);
|
result.setGenerationId(fileId);
|
||||||
|
|
|
||||||
|
|
@ -17,13 +17,13 @@ public class SDf10Table extends AbstractTable<SDf10> {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SDf10 getEntity(Object[] entity) {
|
public SDf10 getEntity(String[] entity) {
|
||||||
SDf10 result = new SDf10();
|
SDf10 result = new SDf10();
|
||||||
result.setOutDocument((String) entity[0]);
|
result.setOutDocument(entity[3]);
|
||||||
result.setDepoCode((String) entity[1]);
|
result.setDepoCode(entity[0]);
|
||||||
result.setQuantity((String) entity[2]);
|
result.setQuantity(entity[1]);
|
||||||
result.setSecurityCode((String) entity[3]);
|
result.setSecurityCode(entity[2]);
|
||||||
result.setClientName((String) entity[4]);
|
result.setClientName(entity[5]);
|
||||||
result.setFileName(filename);
|
result.setFileName(filename);
|
||||||
result.setGenerationTime(Instant.now());
|
result.setGenerationTime(Instant.now());
|
||||||
result.setGenerationId(fileId);
|
result.setGenerationId(fileId);
|
||||||
|
|
|
||||||
|
|
@ -17,9 +17,8 @@ public class SDf13Table extends AbstractTable<SDf13> {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SDf13 getEntity(Object[] entity) {
|
public SDf13 getEntity(String[] entity) {
|
||||||
SDf13 result = new SDf13();
|
SDf13 result = new SDf13();
|
||||||
|
|
||||||
result.setFileName(filename);
|
result.setFileName(filename);
|
||||||
result.setGenerationTime(Instant.now());
|
result.setGenerationTime(Instant.now());
|
||||||
result.setGenerationId(fileId);
|
result.setGenerationId(fileId);
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
package ru.spcex.clearing.swt.importer.logic.stages;
|
package ru.spcex.clearing.swt.importer.logic.stages;
|
||||||
|
|
||||||
import com.linuxense.javadbf.DBFReader;
|
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
|
import ru.spcex.clearing.swt.importer.config.settings.ImportSWTServiceSettings;
|
||||||
|
|
@ -8,6 +7,7 @@ import ru.spcex.clearing.swt.importer.logic.data.ResultContainer;
|
||||||
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
|
import ru.spcex.clearing.swt.importer.logic.data.enums.ETable;
|
||||||
import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
|
import ru.spcex.clearing.swt.importer.logic.data.enums.StageResult;
|
||||||
import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
|
import ru.spcex.clearing.swt.importer.logic.data.tables.AbstractTable;
|
||||||
|
import ru.spcex.clearing.swt.importer.util.SWTReader;
|
||||||
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
|
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
|
||||||
import ru.spcex.platform.utils.log.ExceptionUtils;
|
import ru.spcex.platform.utils.log.ExceptionUtils;
|
||||||
|
|
||||||
|
|
@ -43,17 +43,21 @@ public class ImportToDB extends Stage {
|
||||||
byte[] source = resultContainer.getSwtSource();
|
byte[] source = resultContainer.getSwtSource();
|
||||||
Charset sourceCharset = Charset.forName(settings.getCommon().getEncodingSource());
|
Charset sourceCharset = Charset.forName(settings.getCommon().getEncodingSource());
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
try (InputStream is = new ByteArrayInputStream(source);
|
try (InputStream is = new ByteArrayInputStream(source);
|
||||||
DBFReader dbfReader = new DBFReader(is, sourceCharset)) {
|
SWTReader swtReader = new SWTReader(is, sourceCharset)) {
|
||||||
AbstractTable table = mappingEnumTableObjectTable.get(currTable);
|
AbstractTable table = mappingEnumTableObjectTable.get(currTable);
|
||||||
table.setHazelcastService(hazelcastService);
|
table.setHazelcastService(hazelcastService);
|
||||||
table.setFilename(resultContainer.getSwtFile().getName());
|
table.setFilename(resultContainer.getSwtFile().getName());
|
||||||
Long fileId = hazelcastService.getImdgIdGenerator().nextId();
|
Long fileId = hazelcastService.getImdgIdGenerator().nextId();
|
||||||
table.setFileId(fileId);
|
table.setFileId(fileId);
|
||||||
for (int i = 0; i < dbfReader.getRecordCount(); i++) {
|
int counter = 0;
|
||||||
Object[] entity = dbfReader.nextRecord();
|
while (swtReader.hasNextRecord()) {
|
||||||
|
counter++;
|
||||||
|
String[] entity = swtReader.nextRecord();
|
||||||
if (entity == null) {
|
if (entity == null) {
|
||||||
log.warn("record index {} null", i);
|
log.warn("record index {} null", counter);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
table.injectEntity(table.getEntity(entity));
|
table.injectEntity(table.getEntity(entity));
|
||||||
|
|
|
||||||
|
|
@ -25,13 +25,13 @@ public class LoadFileFromDisk extends Stage {
|
||||||
Objects.requireNonNull(resultContainer.getSwtFile());
|
Objects.requireNonNull(resultContainer.getSwtFile());
|
||||||
UUID taskUuid = resultContainer.getUuid();
|
UUID taskUuid = resultContainer.getUuid();
|
||||||
|
|
||||||
File dbfFile = resultContainer.getSwtFile();
|
File swtFile = resultContainer.getSwtFile();
|
||||||
byte[] fileBytes;
|
byte[] fileBytes;
|
||||||
try {
|
try {
|
||||||
log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getSwtFile().getName());
|
log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getSwtFile().getName());
|
||||||
fileBytes = Files.readAllBytes(Paths.get(dbfFile.getAbsolutePath()));
|
fileBytes = Files.readAllBytes(Paths.get(swtFile.getAbsolutePath()));
|
||||||
if (fileBytes.length == 0) throw new IOException("Empty file");
|
if (fileBytes.length == 0) throw new IOException("Empty file");
|
||||||
if (settings.getStore().isDeleteSrcFiles()) Files.delete(dbfFile.toPath());
|
if (settings.getStore().isDeleteSrcFiles()) Files.delete(swtFile.toPath());
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getSwtFile().getName()), e);
|
log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getSwtFile().getName()), e);
|
||||||
return StageResult.ERROR;
|
return StageResult.ERROR;
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ public class ValidateFields extends Stage implements InitializingBean {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 1. Проверяет кодировку из настройки dbf.encoding-source
|
* 1. Проверяет кодировку из настройки swt.encoding-source
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void afterPropertiesSet() throws Exception {
|
public void afterPropertiesSet() throws Exception {
|
||||||
|
|
|
||||||
|
|
@ -23,32 +23,31 @@ public class FileChecker {
|
||||||
Map<ETable, List<File>> newFiles = new EnumMap<>(ETable.class);
|
Map<ETable, List<File>> newFiles = new EnumMap<>(ETable.class);
|
||||||
|
|
||||||
String srcDir = settings.getStore().getSrcDir();
|
String srcDir = settings.getStore().getSrcDir();
|
||||||
List<File> dbfFiles = lsDBF(srcDir);
|
List<File> swtFiles = lsSWT(srcDir);
|
||||||
if (dbfFiles.isEmpty()) return newFiles;
|
if (swtFiles.isEmpty()) return newFiles;
|
||||||
|
|
||||||
for (File dbfFile : dbfFiles) {
|
for (File swtFile : swtFiles) {
|
||||||
if (dbfFile.isDirectory()) continue;
|
if (swtFile.isDirectory()) continue;
|
||||||
|
|
||||||
ETable currTable = ETable.getTableForFilename(dbfFile.getName());
|
ETable currTable = ETable.getTableForFilename(swtFile.getName());
|
||||||
if (currTable == null) continue;
|
if (currTable == null) continue;
|
||||||
if (specificTable != null && !specificTable.equals(currTable)) continue;
|
if (specificTable != null && !specificTable.equals(currTable)) continue;
|
||||||
List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());
|
List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());
|
||||||
currList.add(dbfFile);
|
currList.add(swtFile);
|
||||||
}
|
}
|
||||||
return newFiles;
|
return newFiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<File> lsDBF(String dbfDirPath) {
|
private List<File> lsSWT(String swtDirPath) {
|
||||||
File dbfDir = new File(dbfDirPath);
|
File swtDir = new File(swtDirPath);
|
||||||
File[] dbfFiles = dbfDir.listFiles((dir, name) -> {
|
File[] swtFiles = swtDir.listFiles((dir, name) -> {
|
||||||
int formatPosition = name.lastIndexOf(".");
|
String offsetName = name.substring(0, 9);
|
||||||
if (formatPosition == -1 || formatPosition == name.length() - 1) return false;
|
return offsetName.equalsIgnoreCase("RDC_KS_DF");
|
||||||
return "dbf".equalsIgnoreCase(name.substring(formatPosition + 1));
|
|
||||||
});
|
});
|
||||||
|
|
||||||
List<File> resultFiles = new LinkedList<>();
|
List<File> resultFiles = new LinkedList<>();
|
||||||
if (dbfFiles != null && dbfFiles.length >= 1) {
|
if (swtFiles != null && swtFiles.length >= 1) {
|
||||||
resultFiles.addAll(Arrays.asList(dbfFiles));
|
resultFiles.addAll(Arrays.asList(swtFiles));
|
||||||
}
|
}
|
||||||
|
|
||||||
return resultFiles;
|
return resultFiles;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
package ru.spcex.clearing.swt.importer.util;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.charset.Charset;
|
||||||
|
import java.util.Scanner;
|
||||||
|
|
||||||
|
public class SWTReader implements Closeable {
|
||||||
|
protected InputStream inputStream;
|
||||||
|
protected Scanner scanner;
|
||||||
|
|
||||||
|
public SWTReader(InputStream inputStream, Charset charset) {
|
||||||
|
this.inputStream = inputStream;
|
||||||
|
this.scanner = new Scanner(inputStream, charset);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String[] nextRecord() {
|
||||||
|
String line = scanner.nextLine();
|
||||||
|
String[] separetedLine = line.split(":");
|
||||||
|
if (separetedLine.length < 5) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return separetedLine;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasNextRecord() {
|
||||||
|
return scanner.hasNextLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
inputStream.close();
|
||||||
|
scanner.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue