treshenkov/indic-search-service
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ExcelExporter fixed, JsonExporter added

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2026-05-22 12:22:12 +03:00
parent 8a05605558
commit 74c5c116cf
9 changed files with 287 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java
View file

@ -5,13 +5,14 @@ import java.io.BufferedWriter;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays; import java.util.Arrays;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.mutable.MutableLong;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.util.IndicSearchCommon;
import ru.nbch.indicsearchservice.util.MovableBufferedReader; import ru.nbch.indicsearchservice.util.MovableBufferedReader;
@Component @Component
@ -51,9 +52,8 @@ public class FileAggregatorCommon {
"RU_ALL_ARREARS" "RU_ALL_ARREARS"
); );
void aggregateRuUidFiles(Path aggregatedFile, Map<String, Path> tableFileMap, String memberCode, MutableLong rowCounter) { void aggregateRuUidFiles(BufferedWriter aggregatedFileWriter, Map<String, Path> tableFileMap, String memberCode, MutableLong rowCounter) {
try (BufferedReader ruUidFileReader = Files.newBufferedReader(tableFileMap.get("RU_UID")); try (BufferedReader ruUidFileReader = Files.newBufferedReader(tableFileMap.get("RU_UID"))) {
BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
Map<String, MovableBufferedReader> childTablesReadersMap = new LinkedHashMap<>(); Map<String, MovableBufferedReader> childTablesReadersMap = new LinkedHashMap<>();
for (String tableName : RU_UID_CHILD_TABLES) { for (String tableName : RU_UID_CHILD_TABLES) {
if (!tableFileMap.containsKey(tableName)) { if (!tableFileMap.containsKey(tableName)) {
@ -69,18 +69,21 @@ public class FileAggregatorCommon {
String ruUidLine; String ruUidLine;
while ((ruUidLine = ruUidFileReader.readLine()) != null) { while ((ruUidLine = ruUidFileReader.readLine()) != null) {
List<String> lineList = Arrays.asList(ruUidLine.split(";")); List<String> lineList = Arrays.asList(ruUidLine.split(";"));
int memberCodeIndex = lineList.indexOf("MEMBER_CODE"); Optional<String> foundMemberCode = IndicSearchCommon.getRowValue(lineList, "MEMBER_CODE");
if (memberCodeIndex != -1) {
if (!lineList.get(memberCodeIndex + 1).startsWith(memberCode)) { if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) {
continue; continue;
}
} }
writer.write(ruUidLine); aggregatedFileWriter.write(ruUidLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
Long ruUidSerialNum = Long.parseLong(ruUidLine.split(";")[2]); Optional<String> foundRuUidSerialNum = IndicSearchCommon.getRowValue(lineList, "SERIAL_NUM");
if (foundRuUidSerialNum.isEmpty()) {
throw new RuntimeException("SERIAL_NUM was not found!");
}
Long ruUidSerialNum = Long.parseLong(foundRuUidSerialNum.get());
for (Map.Entry<String, MovableBufferedReader> entry : childTablesReadersMap.entrySet()) { for (Map.Entry<String, MovableBufferedReader> entry : childTablesReadersMap.entrySet()) {
String childTableName = entry.getKey(); String childTableName = entry.getKey();
@ -91,19 +94,29 @@ public class FileAggregatorCommon {
continue; continue;
} }
Long childAccSerialNum = Long.parseLong(childLine.split(";")[3]); List<String> childLineList = Arrays.asList(childLine.split(";"));
Optional<String> foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM");
if (foundChildAccSerialNum.isEmpty()) {
throw new RuntimeException("ACC_SERIAL_NUM was not found!");
}
Long childAccSerialNum = Long.parseLong(foundChildAccSerialNum.get());
if (ruUidSerialNum > childAccSerialNum) { if (ruUidSerialNum > childAccSerialNum) {
while ((childLine = childReader.nextLine()) != null) { while ((childLine = childReader.nextLine()) != null) {
childAccSerialNum = Long.parseLong(childLine.split(";")[3]); childLineList = Arrays.asList(childLine.split(";"));
foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM");
if (foundChildAccSerialNum.isEmpty()) {
throw new RuntimeException("ACC_SERIAL_NUM was not found!");
}
childAccSerialNum = Long.parseLong(foundChildAccSerialNum.get());
if (childAccSerialNum.equals(ruUidSerialNum)) { if (childAccSerialNum.equals(ruUidSerialNum)) {
writer.write(childLine); aggregatedFileWriter.write(childLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
if (childTableName.equals("RU_COLLATERAL")) { if (childTableName.equals("RU_COLLATERAL")) {
aggregateBlocksTableFile(childTableName, aggregatedFile, tableFileMap, memberCode, rowCounter); aggregateBlocksTableFile(childTableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
} }
} }
@ -112,24 +125,29 @@ public class FileAggregatorCommon {
} }
} }
} else if (ruUidSerialNum.equals(childAccSerialNum)) { } else if (ruUidSerialNum.equals(childAccSerialNum)) {
writer.write(childLine); aggregatedFileWriter.write(childLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
if (childTableName.equals("RU_COLLATERAL")) { if (childTableName.equals("RU_COLLATERAL")) {
aggregateBlocksTableFile(childTableName, aggregatedFile, tableFileMap, memberCode, rowCounter); aggregateBlocksTableFile(childTableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
} }
while ((childLine = childReader.nextLine()) != null) { while ((childLine = childReader.nextLine()) != null) {
childAccSerialNum = Long.parseLong(childLine.split(";")[3]); childLineList = Arrays.asList(childLine.split(";"));
foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM");
if (foundChildAccSerialNum.isEmpty()) {
throw new RuntimeException("ACC_SERIAL_NUM was not found!");
}
childAccSerialNum = Long.parseLong(foundChildAccSerialNum.get());
if (childAccSerialNum.equals(ruUidSerialNum)) { if (childAccSerialNum.equals(ruUidSerialNum)) {
writer.write(childLine); aggregatedFileWriter.write(childLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
if (childTableName.equals("RU_COLLATERAL")) { if (childTableName.equals("RU_COLLATERAL")) {
aggregateBlocksTableFile(childTableName, aggregatedFile, tableFileMap, memberCode, rowCounter); aggregateBlocksTableFile(childTableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
} }
} }
@ -149,40 +167,52 @@ public class FileAggregatorCommon {
} }
} }
void aggregateBlocksTableFile(String parentTableName, Path aggregatedFile, Map<String, Path> tableFileMap, String memberCode, MutableLong rowCounter) { void aggregateBlocksTableFile(String parentTableName, BufferedWriter aggregatedFileWriter, Map<String, Path> tableFileMap, String memberCode, MutableLong rowCounter) {
try (BufferedReader parentFileReader = Files.newBufferedReader(tableFileMap.get(parentTableName)); try (BufferedReader parentFileReader = Files.newBufferedReader(tableFileMap.get(parentTableName));
MovableBufferedReader blocksFileReader = new MovableBufferedReader(Files.newBufferedReader(tableFileMap.get(parentTableName + "_BLOCKS"))); MovableBufferedReader blocksFileReader = new MovableBufferedReader(Files.newBufferedReader(tableFileMap.get(parentTableName + "_BLOCKS")))) {
BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
String parentLine; String parentLine;
while ((parentLine = parentFileReader.readLine()) != null) { while ((parentLine = parentFileReader.readLine()) != null) {
List<String> lineList = Arrays.asList(parentLine.split(";")); List<String> parentLineList = Arrays.asList(parentLine.split(";"));
int memberCodeIndex = lineList.indexOf("MEMBER_CODE"); Optional<String> foundMemberCode = IndicSearchCommon.getRowValue(parentLineList, "MEMBER_CODE");
if (memberCodeIndex != -1) {
if (!lineList.get(memberCodeIndex + 1).startsWith(memberCode)) { if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) {
continue; continue;
}
} }
writer.write(parentLine); aggregatedFileWriter.write(parentLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
Long parentSerialNum = Long.parseLong(parentLine.split(";")[2]); Optional<String> foundParentSerialNum = IndicSearchCommon.getRowValue(parentLineList, "SERIAL_NUM");
if (foundParentSerialNum.isEmpty()) {
throw new RuntimeException("SERIAL_NUM was not found!");
}
Long parentSerialNum = Long.parseLong(foundParentSerialNum.get());
String blocksLine = blocksFileReader.getCurrentLine(); String blocksLine = blocksFileReader.getCurrentLine();
if (blocksLine == null) { if (blocksLine == null) {
continue; continue;
} }
Long blocksParentSerialNum = Long.parseLong(blocksLine.split(";")[4]); List<String> childLineList = Arrays.asList(blocksLine.split(";"));
Optional<String> foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM");
if (foundChildParentSerialNum.isEmpty()) {
throw new RuntimeException("PARENT_SERIAL_NUM was not found!");
}
Long blocksParentSerialNum = Long.parseLong(foundChildParentSerialNum.get());
if (parentSerialNum > blocksParentSerialNum) { if (parentSerialNum > blocksParentSerialNum) {
while ((blocksLine = blocksFileReader.nextLine()) != null) { while ((blocksLine = blocksFileReader.nextLine()) != null) {
blocksParentSerialNum = Long.parseLong(blocksLine.split(";")[4]); childLineList = Arrays.asList(blocksLine.split(";"));
foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM");
if (foundChildParentSerialNum.isEmpty()) {
throw new RuntimeException("PARENT_SERIAL_NUM was not found!");
}
blocksParentSerialNum = Long.parseLong(foundChildParentSerialNum.get());
if (blocksParentSerialNum.equals(parentSerialNum)) { if (blocksParentSerialNum.equals(parentSerialNum)) {
writer.write(blocksLine); aggregatedFileWriter.write(blocksLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
} }
@ -191,16 +221,21 @@ public class FileAggregatorCommon {
} }
} }
} else if (parentSerialNum.equals(blocksParentSerialNum)) { } else if (parentSerialNum.equals(blocksParentSerialNum)) {
writer.write(blocksLine); aggregatedFileWriter.write(blocksLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
while ((blocksLine = blocksFileReader.nextLine()) != null) { while ((blocksLine = blocksFileReader.nextLine()) != null) {
blocksParentSerialNum = Long.parseLong(blocksLine.split(";")[4]); childLineList = Arrays.asList(blocksLine.split(";"));
foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM");
if (foundChildParentSerialNum.isEmpty()) {
throw new RuntimeException("PARENT_SERIAL_NUM was not found!");
}
blocksParentSerialNum = Long.parseLong(foundChildParentSerialNum.get());
if (blocksParentSerialNum.equals(parentSerialNum)) { if (blocksParentSerialNum.equals(parentSerialNum)) {
writer.write(blocksLine); aggregatedFileWriter.write(blocksLine);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
} }

46
src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidFileAggregator.java
View file

@ -6,15 +6,16 @@ import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.mutable.MutableLong;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; import ru.nbch.indicsearchservice.config.settings.ApplicationSettings;
import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter; import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.util.IndicSearchCommon;
@Component @Component
public class GutdfFidFileAggregator implements FileAggregator { public class GutdfFidFileAggregator implements FileAggregator {
@ -57,37 +58,34 @@ public class GutdfFidFileAggregator implements FileAggregator {
public Path aggregateFiles(Map<String, Path> tableFileMap, String memberCode, MutableLong rowCounter, UUID taskId) { public Path aggregateFiles(Map<String, Path> tableFileMap, String memberCode, MutableLong rowCounter, UUID taskId) {
Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("AGGREGATED-" + taskId + ".csv"); Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("AGGREGATED-" + taskId + ".csv");
for (String tableName : getRequiredTables()) { try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
if (tableName.equals("RU_UID")) { for (String tableName : getRequiredTables()) {
fileAggregatorCommon.aggregateRuUidFiles(aggregatedFile, tableFileMap, memberCode, rowCounter); if (tableName.equals("RU_UID")) {
} else if (tableName.equals("RU_BANKRUPTCY")) { fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
fileAggregatorCommon.aggregateBlocksTableFile(tableName, aggregatedFile, tableFileMap, memberCode, rowCounter); } else if (tableName.equals("RU_BANKRUPTCY")) {
} else { fileAggregatorCommon.aggregateBlocksTableFile(tableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
try (BufferedReader reader = Files.newBufferedReader(tableFileMap.get(tableName)); } else {
BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { try (BufferedReader reader = Files.newBufferedReader(tableFileMap.get(tableName))) {
String line; String line;
while ((line = reader.readLine()) != null) { while ((line = reader.readLine()) != null) {
List<String> lineList = Arrays.asList(line.split(";")); Optional<String> foundMemberCode = IndicSearchCommon.getRowValue(line, "MEMBER_CODE");
int memberCodeIndex = lineList.indexOf("MEMBER_CODE");
if (memberCodeIndex != -1) { if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) {
if (!lineList.get(memberCodeIndex + 1).startsWith(memberCode)) {
continue; continue;
} }
}
writer.write(line); aggregatedFileWriter.write(line);
writer.newLine(); aggregatedFileWriter.newLine();
rowCounter.increment(); rowCounter.increment();
}
} catch (IOException e) {
throw new RuntimeException(e);
} }
} catch (IOException e) {
throw new RuntimeException(e);
} }
} }
}
try (BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { aggregatedFileWriter.write("EOF");
writer.write("EOF");
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }

50
src/main/java/ru/nbch/indicsearchservice/util/ExcelWriter.java → src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java
View file

@ -1,4 +1,4 @@
package ru.nbch.indicsearchservice.util; package ru.nbch.indicsearchservice.exporter;
import jakarta.annotation.Nullable; import jakarta.annotation.Nullable;
import java.io.BufferedReader; import java.io.BufferedReader;
@ -8,12 +8,11 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set;
import java.util.UUID; import java.util.UUID;
import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.poi.ss.usermodel.BorderStyle; import org.apache.poi.ss.usermodel.BorderStyle;
@ -32,28 +31,17 @@ import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; import ru.nbch.indicsearchservice.config.settings.ApplicationSettings;
@Component @Component
public class ExcelWriter { public class ExcelExporter implements Exporter {
private static final Logger log = LoggerFactory.getLogger(ExcelWriter.class); private static final Logger log = LoggerFactory.getLogger(ExcelExporter.class);
private static final Long EXCEL_SHEET_MAX_SIZE = 1_048_575L; private static final Long EXCEL_SHEET_MAX_SIZE = 1_048_575L;
private static final Set<String> FIRST_LEVEL_TABLES = Set.of(
"RU_UID",
"RU_APPLICATION",
"RU_OBLIGPARTTAKE",
"RU_BANKRUPTCY",
"RU_SETTLED",
"RU_OTHERLEGAL",
"RU_CAPABILITY",
"RU_DEBT_BURDEN_INFO"
);
private final ApplicationSettings settings; private final ApplicationSettings settings;
public ExcelWriter(ApplicationSettings settings) { public ExcelExporter(ApplicationSettings settings) {
this.settings = settings; this.settings = settings;
} }
public void writeExcelFile(Path aggregatedFile, UUID taskId) { public void writeExcelFile(Path aggregatedFile, UUID taskId, Long totalRows) {
Path excelFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".xlsx"); Path excelFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".xlsx");
try (OutputStream os = Files.newOutputStream(excelFile); try (OutputStream os = Files.newOutputStream(excelFile);
@ -67,16 +55,21 @@ public class ExcelWriter {
MutableLong sheetRowCounter = new MutableLong(0L); MutableLong sheetRowCounter = new MutableLong(0L);
MutableLong sheetTableCounter = new MutableLong(1L); MutableLong sheetTableCounter = new MutableLong(1L);
long count = 0L;
boolean isFirst = true; boolean isFirst = true;
String lastTableName = null; String lastTableName;
List<String> rows = new ArrayList<>(); List<String> rows = new ArrayList<>();
while (iterator.hasNext()) { while (iterator.hasNext()) {
String line = iterator.next(); String line = iterator.next();
List<String> lineList = Arrays.asList(line.split(";"));
if (!isFirst && (FIRST_LEVEL_TABLES.contains(line.split(";")[0]) || line.equals("EOF"))) { log.info("Processing row: {} / {}", count++, totalRows);
if (!isFirst && (FIRST_LEVEL_TABLES.contains(lineList.getFirst()) || line.equals("EOF"))) {
lastTableName = rows.getFirst().split(";")[0]; lastTableName = rows.getFirst().split(";")[0];
writeToExcel(workbook, rows, sheetRowCounter, sheetTableCounter, headerCellStyle, cellStyle); writeToExcel(workbook, rows, sheetRowCounter, sheetTableCounter, headerCellStyle, cellStyle);
if (lastTableName != null && !lastTableName.equals(line.split(";")[0])) { if (lastTableName != null && !lastTableName.equals(lineList.getFirst())) {
sheetRowCounter.setValue(0L); sheetRowCounter.setValue(0L);
sheetTableCounter.setValue(1L); sheetTableCounter.setValue(1L);
} }
@ -106,21 +99,26 @@ public class ExcelWriter {
String parentTableName = rows.getFirst().split(";")[0]; String parentTableName = rows.getFirst().split(";")[0];
Sheet tableSheet; Sheet tableSheet;
if (sheetRowCounter.longValue() + (rows.size() * 2L) >= EXCEL_SHEET_MAX_SIZE) { int rowsApproximateSize = rows.stream().mapToInt(r -> r.split(";", -1).length).sum();
if (sheetRowCounter.longValue() + (rowsApproximateSize * 2L) >= EXCEL_SHEET_MAX_SIZE) {
tableSheet = createSheet(workbook, parentTableName.toUpperCase() + "_" + sheetTableCounter.incrementAndGet()); tableSheet = createSheet(workbook, parentTableName.toUpperCase() + "_" + sheetTableCounter.incrementAndGet());
sheetRowCounter.setValue(0L); sheetRowCounter.setValue(0L);
} else { } else {
tableSheet = createSheet(workbook, parentTableName.toUpperCase() + "_" + sheetTableCounter.longValue()); tableSheet = createSheet(workbook, parentTableName.toUpperCase());
} }
for (String row : rows) { for (String row : rows) {
List<String> rowList = Arrays.asList(row.split(";", -1)); List<String> rowList = Arrays.asList(row.split(";", -1));
String tableName = rowList.getFirst(); String tableName = rowList.getFirst();
List<String> rowSublist = rowList.subList(5, rowList.size()); List<String> rowSublist = rowList.subList(1, rowList.size());
Map<String, String> rowValues = new HashMap<>(); Map<String, String> rowValues = new LinkedHashMap<>();
for (int i = 0; i + 1 <= rowSublist.size(); i = i + 2) { for (int i = 0; i + 1 <= rowSublist.size(); i = i + 2) {
rowValues.put(rowSublist.get(i), rowSublist.get(i + 1)); try {
rowValues.put(rowSublist.get(i), rowSublist.get(i + 1));
} catch (Exception e) {
throw new RuntimeException(e);
}
} }
createHeaderRow( createHeaderRow(

16
src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java Normal file
View file

@ -0,0 +1,16 @@
package ru.nbch.indicsearchservice.exporter;
import java.util.Set;
public interface Exporter {
Set<String> FIRST_LEVEL_TABLES = Set.of(
"RU_UID",
"RU_APPLICATION",
"RU_OBLIGPARTTAKE",
"RU_BANKRUPTCY",
"RU_SETTLED",
"RU_OTHERLEGAL",
"RU_CAPABILITY",
"RU_DEBT_BURDEN_INFO"
);
}

113
src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java Normal file
View file

@ -0,0 +1,113 @@
package ru.nbch.indicsearchservice.exporter;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.config.settings.ApplicationSettings;
import ru.nbch.indicsearchservice.model.TableData;
import tools.jackson.core.JsonGenerator;
import tools.jackson.databind.json.JsonMapper;
@Component
public class JsonExporter implements Exporter {
private static final Logger log = LoggerFactory.getLogger(JsonExporter.class);
private final ApplicationSettings settings;
private final JsonMapper jsonMapper;
public JsonExporter(ApplicationSettings settings,
JsonMapper jsonMapper) {
this.settings = settings;
this.jsonMapper = jsonMapper;
}
public void writeJsonFile(Path aggregatedFile, UUID taskId) {
Path jsonFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".json");
try (BufferedReader reader = Files.newBufferedReader(aggregatedFile);
JsonGenerator jsonGenerator = jsonMapper.createGenerator(Files.newOutputStream(jsonFile))) {
jsonGenerator.writeStartArray();
Iterator<String> iterator = reader.lines().iterator();
List<String> rows = new ArrayList<>();
boolean isFirst = true;
while (iterator.hasNext()) {
String line = iterator.next();
if (line == null || line.trim().isEmpty()) {
continue;
}
List<String> lineList = Arrays.asList(line.split(";", -1));
String tableName = lineList.getFirst();
if (!isFirst && (FIRST_LEVEL_TABLES.contains(tableName) || line.equals("EOF"))) {
TableData parentData = buildTableDataHierarchy(rows);
if (parentData != null) {
jsonMapper.writeValue(jsonGenerator, parentData);
}
rows.clear();
}
rows.add(line);
isFirst = false;
}
jsonGenerator.writeEndArray();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
private TableData buildTableDataHierarchy(List<String> rows) {
if (rows == null || rows.isEmpty()) {
return null;
}
TableData parentTableData = parseSingleRow(rows.getFirst());
if (parentTableData == null) {
return null;
}
for (int i = 1; i < rows.size(); i++) {
TableData childTableData = parseSingleRow(rows.get(i));
if (childTableData != null) {
parentTableData.getRelatedTableData().add(childTableData);
}
}
return parentTableData;
}
private TableData parseSingleRow(String row) {
String[] parts = row.split(";", -1);
if (parts.length == 0 || parts[0].isEmpty()) {
return null;
}
String tableName = parts[0];
List<TableData.FieldData> fieldData = new ArrayList<>();
for (int i = 1; i + 1 < parts.length; i += 2) {
String columnName = parts[i];
String columnValue = parts[i + 1];
if (columnName != null && !columnName.isEmpty()) {
fieldData.add(new TableData.FieldData(columnName, columnValue));
}
}
return new TableData(tableName, fieldData);
}
}

2
src/main/java/ru/nbch/indicsearchservice/model/TableData.java
View file

@ -1,9 +1,11 @@
package ru.nbch.indicsearchservice.model; package ru.nbch.indicsearchservice.model;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
@JsonPropertyOrder({"tableName", "fieldData", "relatedTableData"})
public class TableData { public class TableData {
private String tableName; private String tableName;
private List<FieldData> fieldData = new ArrayList<>(); private List<FieldData> fieldData = new ArrayList<>();

27
src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java
View file

@ -19,34 +19,33 @@ import ru.nbch.indicsearchservice.aggregator.FileAggregator;
import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto; import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto;
import ru.nbch.indicsearchservice.dto.response.AddSearchJobResponseDto; import ru.nbch.indicsearchservice.dto.response.AddSearchJobResponseDto;
import ru.nbch.indicsearchservice.dto.response.JobStatusResponseDto; import ru.nbch.indicsearchservice.dto.response.JobStatusResponseDto;
import ru.nbch.indicsearchservice.exporter.ExcelExporter;
import ru.nbch.indicsearchservice.exporter.JsonExporter;
import ru.nbch.indicsearchservice.loader.TableLoader; import ru.nbch.indicsearchservice.loader.TableLoader;
import ru.nbch.indicsearchservice.util.ExcelWriter;
import ru.nbch.indicsearchservice.util.IndicSearchUtils; import ru.nbch.indicsearchservice.util.IndicSearchUtils;
import tools.jackson.databind.json.JsonMapper;
@Service @Service
public class IndicSearchService { public class IndicSearchService {
private static final Logger log = LoggerFactory.getLogger(IndicSearchService.class); private static final Logger log = LoggerFactory.getLogger(IndicSearchService.class);
private static final boolean IS_EXPORT_AS_EXCEL = true; private static final boolean IS_EXPORT_AS_EXCEL = false;
private final List<TableLoader> tableLoaders; private final List<TableLoader> tableLoaders;
private final List<FileAggregator> fileAggregators; private final List<FileAggregator> fileAggregators;
private final IndicSearchUtils indicSearchUtils; private final IndicSearchUtils indicSearchUtils;
private final ExcelWriter excelWriter; private final ExcelExporter excelExporter;
private final JsonExporter jsonExporter;
private final JsonMapper jsonMapper;
public IndicSearchService(List<TableLoader> tableLoaders, public IndicSearchService(List<TableLoader> tableLoaders,
List<FileAggregator> fileAggregators, List<FileAggregator> fileAggregators,
IndicSearchUtils indicSearchUtils, IndicSearchUtils indicSearchUtils,
ExcelWriter excelWriter, ExcelExporter excelExporter,
JsonMapper jsonMapper) { JsonExporter jsonExporter) {
this.tableLoaders = tableLoaders; this.tableLoaders = tableLoaders;
this.fileAggregators = fileAggregators; this.fileAggregators = fileAggregators;
this.indicSearchUtils = indicSearchUtils; this.indicSearchUtils = indicSearchUtils;
this.excelWriter = excelWriter; this.excelExporter = excelExporter;
this.jsonMapper = jsonMapper; this.jsonExporter = jsonExporter;
} }
public AddSearchJobResponseDto addSearchJob(AddSearchJobRequestDto request) { public AddSearchJobResponseDto addSearchJob(AddSearchJobRequestDto request) {
@ -112,13 +111,9 @@ public class IndicSearchService {
sw.start(); sw.start();
if (!IS_EXPORT_AS_EXCEL) { if (!IS_EXPORT_AS_EXCEL) {
// try { jsonExporter.writeJsonFile(aggregatedFile, taskId);
// Files.write(Path.of("1.json"), jsonMapper.writeValueAsBytes(result));
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
} else { } else {
excelWriter.writeExcelFile(aggregatedFile, taskId); excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue());
} }
sw.stop(); sw.stop();

19
src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java Normal file
View file

@ -0,0 +1,19 @@
package ru.nbch.indicsearchservice.util;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
public class IndicSearchCommon {
public static Optional<String> getRowValue(String string, String key) {
return getRowValue(Arrays.asList(string.split(";", -1)), key);
}
public static Optional<String> getRowValue(List<String> stringValues, String key) {
int keyIndex = stringValues.indexOf(key);
if (keyIndex != -1) {
return Optional.of(stringValues.get(keyIndex + 1));
}
return Optional.empty();
}
}

78
src/main/java/ru/nbch/indicsearchservice/util/IndicSearchUtils.java
View file

@ -5,14 +5,8 @@ import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.sql.Date;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
@ -20,7 +14,6 @@ import java.util.function.Function;
import org.apache.ibatis.cursor.Cursor; import org.apache.ibatis.cursor.Cursor;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; import ru.nbch.indicsearchservice.config.settings.ApplicationSettings;
import ru.nbch.indicsearchservice.model.TableData;
import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter; import ru.nbch.indicsearchservice.model.enums.SearchParameter;
@ -44,30 +37,6 @@ public class IndicSearchUtils {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append(tableName).append(";"); sb.append(tableName).append(";");
if (row.containsKey("FID") && row.get("FID") != null) {
sb.append(row.get("FID").toString()).append(";");
} else {
sb.append(";");
}
if (row.containsKey("SERIAL_NUM") && row.get("SERIAL_NUM") != null) {
sb.append(row.get("SERIAL_NUM").toString()).append(";");
} else {
sb.append(";");
}
if (row.containsKey("ACC_SERIAL_NUM") && row.get("ACC_SERIAL_NUM") != null) {
sb.append(row.get("ACC_SERIAL_NUM").toString()).append(";");
} else {
sb.append(";");
}
if (row.containsKey("PARENT_SERIAL_NUM") && row.get("PARENT_SERIAL_NUM") != null) {
sb.append(row.get("PARENT_SERIAL_NUM").toString()).append(";");
} else {
sb.append(";");
}
for (Map.Entry<String, Object> entry : row.entrySet()) { for (Map.Entry<String, Object> entry : row.entrySet()) {
String columnName = entry.getKey(); String columnName = entry.getKey();
Object columnValue = entry.getValue(); Object columnValue = entry.getValue();
@ -111,51 +80,4 @@ public class IndicSearchUtils {
return fids; return fids;
} }
public TableData mapToTableData(String table, Map<String, Object> row) {
List<TableData.FieldData> fields = row.entrySet().stream()
.map(e -> new TableData.FieldData(e.getKey(), parseTableValue(e.getValue())))
.toList();
return new TableData(table, fields);
}
private Object parseTableValue(Object value) {
return switch (value) {
case Timestamp timestamp ->
timestamp.toLocalDateTime().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
case Date date -> {
LocalDate localDate = date.toLocalDate();
if (localDate.equals(Date.valueOf("0001-01-01").toLocalDate()) ||
localDate.equals(Date.valueOf("0001-01-03").toLocalDate())) {
yield null;
}
yield localDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
}
case Integer i -> {
if (i == Integer.MAX_VALUE) {
yield null;
}
yield i;
}
case Long l -> {
if (l == Long.MAX_VALUE) {
yield null;
}
yield l;
}
case null -> null;
default -> value;
};
}
public List<TableData> getFirstLevelTables(Map<String, List<TableData>> tableDataMapByFid) {
List<TableData> result = new ArrayList<>();
for (Map.Entry<String, List<TableData>> entry : tableDataMapByFid.entrySet()) {
if (entry.getKey().equals("RU_UID")) {}
}
return result;
}
} }