treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dbf importer, work with sonar

Browse source
This commit is contained in:
akulikov 2022-08-11 13:44:05 +03:00
parent a07a31914b
commit 2c01a624af
7 changed files with 30 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/DBFLoaderApplication.java
View file

@ -10,7 +10,7 @@ public class DBFLoaderApplication {
try { try {
SpringApplicationBuilder builder = new SpringApplicationBuilder(DBFLoaderApplication.class); SpringApplicationBuilder builder = new SpringApplicationBuilder(DBFLoaderApplication.class);
builder.run(args); builder.run(args);
} catch (Throwable e) { } catch (Exception e) {
LoggerFactory.getLogger(DBFLoaderApplication.class).error("DBF-Loader start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage()); LoggerFactory.getLogger(DBFLoaderApplication.class).error("DBF-Loader start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage());
System.exit(-1); System.exit(-1);
} }

4
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/data/DBStructureDBF.java
View file

@ -2,11 +2,11 @@ package ru.spcex.clearing.dbf.importer.logic.data;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import java.util.HashMap; import java.util.EnumMap;
import java.util.Map; import java.util.Map;
public class DBStructureDBF { public class DBStructureDBF {
private Map<Table, TableStructureDBF> tables = new HashMap<>(); private Map<Table, TableStructureDBF> tables = new EnumMap<>(Table.class);
public Map<Table, TableStructureDBF> getTables() { public Map<Table, TableStructureDBF> getTables() {
return tables; return tables;

46
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ImportToDB.java
View file

@ -75,32 +75,27 @@ public class ImportToDB extends Stage {
if (batchIdx == batchCount && batchRem == 0) continue; if (batchIdx == batchCount && batchRem == 0) continue;
final Integer currentBatchIdx = batchIdx; final Integer currentBatchIdx = batchIdx;
try { int[] rowsInserted = dbfJdbcTemplate.batchUpdate(
int[] rowsInserted = dbfJdbcTemplate.batchUpdate( "insert into " + currTable.name() +
"insert into " + currTable.name() + " (" + String.join(",", columnNames) + ")" +
" (" + String.join(",", columnNames) + ")" + " values (" + String.join(",", Collections.nCopies(columnCount, "?")) + ")",
" values (" + String.join(",", Collections.nCopies(columnCount, "?")) + ")", new BatchPreparedStatementSetter() {
new BatchPreparedStatementSetter() { @Override
@Override public void setValues(PreparedStatement ps, int i) throws SQLException {
public void setValues(PreparedStatement ps, int i) throws SQLException { Object[] currValuesRow = values.get(currentBatchIdx * maxBatchSize + i);
Object[] currValuesRow = values.get(currentBatchIdx * maxBatchSize + i); int parameterIndex = 1;
int parameterIndex = 1; for (Object object : currValuesRow) {
for (Object object : currValuesRow) { ps.setObject(parameterIndex, object, columnTypes.get(parameterIndex - 1));
ps.setObject(parameterIndex, object, columnTypes.get(parameterIndex - 1)); parameterIndex++;
parameterIndex++;
}
} }
}
@Override @Override
public int getBatchSize() { public int getBatchSize() {
return currentBatchIdx == batchCount ? batchRem : maxBatchSize; return currentBatchIdx == batchCount ? batchRem : maxBatchSize;
} }
}); });
log.info("uuid {}. Imported records: {}", resultContainer.getUuid(), rowsInserted.length); log.info("uuid {}. Imported records: {}", resultContainer.getUuid(), rowsInserted.length);
} catch (DataAccessException e) {
log.error(String.format("uuid %s. Can't insert records", resultContainer.getUuid()), e);
return StageResult.ERROR;
}
} }
} catch (IOException e) { } catch (IOException e) {
@ -108,6 +103,9 @@ public class ImportToDB extends Stage {
resultContainer.getUuid(), resultContainer.getUuid(),
resultContainer.getDbfFile().getName()), e); resultContainer.getDbfFile().getName()), e);
return StageResult.ERROR; return StageResult.ERROR;
} catch (DataAccessException e) {
log.error(String.format("uuid %s. Can't insert records", resultContainer.getUuid()), e);
return StageResult.ERROR;
} }

9
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/LoadFileFromDisk.java
View file

@ -9,6 +9,7 @@ import ru.spcex.clearing.dbf.importer.properties.AProperties;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.Objects; import java.util.Objects;
import java.util.UUID; import java.util.UUID;
@ -32,13 +33,7 @@ public class LoadFileFromDisk extends Stage {
log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getDbfFile().getName()); log.debug("uuid {}. Read all bytes from source file {}", taskUuid, resultContainer.getDbfFile().getName());
fileBytes = Files.readAllBytes(Paths.get(dbfFile.getAbsolutePath())); fileBytes = Files.readAllBytes(Paths.get(dbfFile.getAbsolutePath()));
if (fileBytes.length == 0) throw new IOException("Empty file"); if (fileBytes.length == 0) throw new IOException("Empty file");
if (properties.deleteSrcFiles()) { if (properties.deleteSrcFiles()) Files.delete(Path.of(dbfFile.getPath()));
boolean deleteOk = dbfFile.delete();
if (!deleteOk) {
log.warn("uuid {}. Can't remove source file {}.", taskUuid, dbfFile.getPath());
return StageResult.ERROR;
}
}
} catch (IOException e) { } catch (IOException e) {
log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getDbfFile().getName()), e); log.error(String.format("uuid %s. Can't read file %s", taskUuid, resultContainer.getDbfFile().getName()), e);
return StageResult.ERROR; return StageResult.ERROR;

2
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/logic/stages/ValidateFields.java
View file

@ -95,7 +95,7 @@ public class ValidateFields extends Stage implements InitializingBean {
taskUuid, taskUuid,
currFieldName, currFieldName,
currTable, currTable,
dbfDataType.name(), dbfDataType != null ? dbfDataType.name() : "null",
currColumnStructure.getType()); currColumnStructure.getType());
valid = false; valid = false;
continue; continue;

4
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/DBFImporterService.java
View file

@ -8,7 +8,6 @@ import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer; import ru.spcex.clearing.dbf.importer.logic.data.ResultContainer;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.logic.stages.Processor; import ru.spcex.clearing.dbf.importer.logic.stages.Processor;
import ru.spcex.clearing.dbf.importer.properties.AProperties;
import java.io.File; import java.io.File;
import java.util.List; import java.util.List;
@ -21,8 +20,7 @@ public class DBFImporterService {
private final ThreadPoolTaskExecutor executorService; private final ThreadPoolTaskExecutor executorService;
private final Processor processor; private final Processor processor;
public DBFImporterService(AProperties properties, public DBFImporterService(@Qualifier("fileChecker") FileChecker messageListener,
@Qualifier("fileChecker") FileChecker messageListener,
@Qualifier("executor") ThreadPoolTaskExecutor executorService, @Qualifier("executor") ThreadPoolTaskExecutor executorService,
@Qualifier("processor") Processor processor) { @Qualifier("processor") Processor processor) {
this.fileChecker = messageListener; this.fileChecker = messageListener;

8
clearing-parent/dbf-importer/src/main/java/ru/spcex/clearing/dbf/importer/services/FileChecker.java
View file

@ -1,7 +1,5 @@
package ru.spcex.clearing.dbf.importer.services; package ru.spcex.clearing.dbf.importer.services;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.spcex.clearing.dbf.importer.logic.data.enums.Table; import ru.spcex.clearing.dbf.importer.logic.data.enums.Table;
import ru.spcex.clearing.dbf.importer.properties.AProperties; import ru.spcex.clearing.dbf.importer.properties.AProperties;
@ -11,7 +9,6 @@ import java.util.*;
@Service("fileChecker") @Service("fileChecker")
public class FileChecker { public class FileChecker {
private final Logger log = LoggerFactory.getLogger(getClass());
private final AProperties properties; private final AProperties properties;
public FileChecker(AProperties properties) { public FileChecker(AProperties properties) {
@ -19,7 +16,7 @@ public class FileChecker {
} }
public Map<Table, List<File>> checkNewFiles() { public Map<Table, List<File>> checkNewFiles() {
Map<Table, List<File>> newFiles = new HashMap<>(); Map<Table, List<File>> newFiles = new EnumMap<>(Table.class);
String srcDir = properties.getSrcDir(); String srcDir = properties.getSrcDir();
List<File> dbfFiles = lsDBF(srcDir); List<File> dbfFiles = lsDBF(srcDir);
@ -31,11 +28,8 @@ public class FileChecker {
Table currTable = Table.getTableForFilename(dbfFile.getName()); Table currTable = Table.getTableForFilename(dbfFile.getName());
if (currTable == null) continue; if (currTable == null) continue;
List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>()); List<File> currList = newFiles.computeIfAbsent(currTable, list -> new LinkedList<>());
currList.add(dbfFile); currList.add(dbfFile);
} }
return newFiles; return newFiles;