treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dbf importer, work with sonar

Browse source
This commit is contained in:
akulikov 2022-08-11 13:44:05 +03:00
parent a07a31914b
commit 2c01a624af
7 changed files with 30 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/DBFLoaderApplication.java
View file

@ -10,7 +10,7 @@ public class DBFLoaderApplication {
try {
SpringApplicationBuilder builder = new SpringApplicationBuilder(DBFLoaderApplication.class);
builder.run(args);
} catch (Throwable e) {
} catch (Exception e) {
LoggerFactory.getLogger(DBFLoaderApplication.class).error("DBF-Loader start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage());
System.exit(-1);
}

4
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java
View file

@ -2,11 +2,11 @@ package ru.spcex.clearing.dbf.importer.logic.data;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import java.util.HashMap;
import java.util.EnumMap;
import java.util.Map;
public class DBStructureDBF {
private Map<Table, TableStructureDBF> tables = new HashMap<>();
private Map<Table, TableStructureDBF> tables = new EnumMap<>(Table.class);
public Map<Table, TableStructureDBF> getTables() {
return tables;

46
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java
View file

@ -75,32 +75,27 @@ public class ImportToDB extends Stage {
if (batchIdx == batchCount && batchRem == 0) continue;
final Integer currentBatchIdx = batchIdx;
try {
int[] rowsInserted = dbfJdbcTemplate.batchUpdate(
"insert into " + currTable.name() +
" (" + String.join(",", columnNames) + ")" +
" values (" + String.join(",", Collections.nCopies(columnCount, "?")) + ")",
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
Object[] currValuesRow = values.get(currentBatchIdx * maxBatchSize + i);
int parameterIndex = 1;
for (Object object : currValuesRow) {
ps.setObject(parameterIndex, object, columnTypes.get(parameterIndex - 1));
parameterIndex++;
}
int[] rowsInserted = dbfJdbcTemplate.batchUpdate(
"insert into " + currTable.name() +
" (" + String.join(",", columnNames) + ")" +
" values (" + String.join(",", Collections.nCopies(columnCount, "?")) + ")",
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
Object[] currValuesRow = values.get(currentBatchIdx * maxBatchSize + i);
int parameterIndex = 1;
for (Object object : currValuesRow) {
ps.setObject(parameterIndex, object, columnTypes.get(parameterIndex - 1));
parameterIndex++;
}
}
@Override
public int getBatchSize() {
return currentBatchIdx == batchCount ? batchRem : maxBatchSize;
}
});
log.info("uuid {}. Imported records: {}", resultContainer.getUuid(), rowsInserted.length);
} catch (DataAccessException e) {
log.error(String.format("uuid %s. Can't insert records", resultContainer.getUuid()), e);
return StageResult.ERROR;
}
@Override
public int getBatchSize() {
return currentBatchIdx == batchCount ? batchRem : maxBatchSize;
}
});
log.info("uuid {}. Imported records: {}", resultContainer.getUuid(), rowsInserted.length);
}
} catch (IOException e) {
@ -108,6 +103,9 @@ public class ImportToDB extends Stage {
resultContainer.getUuid(),
resultContainer.getDbfFile().getName()), e);
return StageResult.ERROR;
} catch (DataAccessException e) {
log.error(String.format("uuid %s. Can't insert records", resultContainer.getUuid()), e);
return StageResult.ERROR;
}

9
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/LoadFileFromDisk.java
View file

@ -9,6 +9,7 @@ import ru.spcex.clearing.dbf.importer.properties.AProperties;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.UUID;
@ -32,13 +33,7 @@ public class LoadFileFromDisk extends Stage {
log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getDbfFile().getName());
fileBytes = Files.readAllBytes(Paths.get(dbfFile.getAbsolutePath()));
if (fileBytes.length == 0) throw new IOException("Empty file");
if (properties.deleteSrcFiles()) {
boolean deleteOk = dbfFile.delete();
if (!deleteOk) {
log.warn("uuid {}. Can't remove source file {}.", taskUuid, dbfFile.getPath());
return StageResult.ERROR;
}
}
if (properties.deleteSrcFiles()) Files.delete(Path.of(dbfFile.getPath()));
} catch (IOException e) {
log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getDbfFile().getName()), e);
return StageResult.ERROR;

2
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java
View file

@ -95,7 +95,7 @@ public class ValidateFields extends Stage implements InitializingBean {
taskUuid,
currFieldName,
currTable,
dbfDataType.name(),
dbfDataType != null ? dbfDataType.name() : "null",
currColumnStructure.getType());
valid = false;
continue;

4
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java
View file

@ -8,7 +8,6 @@ import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.stages.Processor;
import ru.spcex.clearing.dbf.importer.properties.AProperties;
import java.io.File;
import java.util.List;
@ -21,8 +20,7 @@ public class DBFImporterService {
private final ThreadPoolTaskExecutor executorService;
private final Processor processor;
public DBFImporterService(AProperties properties,
@Qualifier("fileChecker") FileChecker messageListener,
public DBFImporterService(@Qualifier("fileChecker") FileChecker messageListener,
@Qualifier("executor") ThreadPoolTaskExecutor executorService,
@Qualifier("processor") Processor processor) {
this.fileChecker = messageListener;

8
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java
View file

@ -1,7 +1,5 @@
package ru.spcex.clearing.dbf.importer.services;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.properties.AProperties;
@ -11,7 +9,6 @@ import java.util.*;
@Service("fileChecker")
public class FileChecker {
private final Logger log = LoggerFactory.getLogger(getClass());
private final AProperties properties;
public FileChecker(AProperties properties) {
@ -19,7 +16,7 @@ public class FileChecker {
}
public Map<Table, List<File>> checkNewFiles() {
Map<Table, List<File>> newFiles = new HashMap<>();
Map<Table, List<File>> newFiles = new EnumMap<>(Table.class);
String srcDir = properties.getSrcDir();
List<File> dbfFiles = lsDBF(srcDir);
@ -31,11 +28,8 @@ public class FileChecker {
Table currTable = Table.getTableForFilename(dbfFile.getName());
if (currTable == null) continue;
List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());
currList.add(dbfFile);
}
return newFiles;