diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java index 1512c00..2e062ae 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java @@ -5,13 +5,14 @@ import java.io.BufferedWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.mutable.MutableLong; import org.springframework.stereotype.Component; +import ru.nbch.indicsearchservice.util.IndicSearchCommon; import ru.nbch.indicsearchservice.util.MovableBufferedReader; @Component @@ -51,9 +52,8 @@ public class FileAggregatorCommon { "RU_ALL_ARREARS" ); - void aggregateRuUidFiles(Path aggregatedFile, Map tableFileMap, String memberCode, MutableLong rowCounter) { - try (BufferedReader ruUidFileReader = Files.newBufferedReader(tableFileMap.get("RU_UID")); - BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + void aggregateRuUidFiles(BufferedWriter aggregatedFileWriter, Map tableFileMap, String memberCode, MutableLong rowCounter) { + try (BufferedReader ruUidFileReader = Files.newBufferedReader(tableFileMap.get("RU_UID"))) { Map childTablesReadersMap = new LinkedHashMap<>(); for (String tableName : RU_UID_CHILD_TABLES) { if (!tableFileMap.containsKey(tableName)) { @@ -69,18 +69,21 @@ public class FileAggregatorCommon { String ruUidLine; while ((ruUidLine = ruUidFileReader.readLine()) != null) { List lineList = Arrays.asList(ruUidLine.split(";")); - int memberCodeIndex = lineList.indexOf("MEMBER_CODE"); - if (memberCodeIndex != -1) { - if (!lineList.get(memberCodeIndex + 1).startsWith(memberCode)) { - continue; - } + Optional foundMemberCode = IndicSearchCommon.getRowValue(lineList, "MEMBER_CODE"); + + if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) { + continue; } - writer.write(ruUidLine); - writer.newLine(); + aggregatedFileWriter.write(ruUidLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); - Long ruUidSerialNum = Long.parseLong(ruUidLine.split(";")[2]); + Optional foundRuUidSerialNum = IndicSearchCommon.getRowValue(lineList, "SERIAL_NUM"); + if (foundRuUidSerialNum.isEmpty()) { + throw new RuntimeException("SERIAL_NUM was not found!"); + } + Long ruUidSerialNum = Long.parseLong(foundRuUidSerialNum.get()); for (Map.Entry entry : childTablesReadersMap.entrySet()) { String childTableName = entry.getKey(); @@ -91,19 +94,29 @@ public class FileAggregatorCommon { continue; } - Long childAccSerialNum = Long.parseLong(childLine.split(";")[3]); + List childLineList = Arrays.asList(childLine.split(";")); + Optional foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM"); + if (foundChildAccSerialNum.isEmpty()) { + throw new RuntimeException("ACC_SERIAL_NUM was not found!"); + } + Long childAccSerialNum = Long.parseLong(foundChildAccSerialNum.get()); if (ruUidSerialNum > childAccSerialNum) { while ((childLine = childReader.nextLine()) != null) { - childAccSerialNum = Long.parseLong(childLine.split(";")[3]); + childLineList = Arrays.asList(childLine.split(";")); + foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM"); + if (foundChildAccSerialNum.isEmpty()) { + throw new RuntimeException("ACC_SERIAL_NUM was not found!"); + } + childAccSerialNum = Long.parseLong(foundChildAccSerialNum.get()); if (childAccSerialNum.equals(ruUidSerialNum)) { - writer.write(childLine); - writer.newLine(); + aggregatedFileWriter.write(childLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); if (childTableName.equals("RU_COLLATERAL")) { - aggregateBlocksTableFile(childTableName, aggregatedFile, tableFileMap, memberCode, rowCounter); + aggregateBlocksTableFile(childTableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter); } } @@ -112,24 +125,29 @@ public class FileAggregatorCommon { } } } else if (ruUidSerialNum.equals(childAccSerialNum)) { - writer.write(childLine); - writer.newLine(); + aggregatedFileWriter.write(childLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); if (childTableName.equals("RU_COLLATERAL")) { - aggregateBlocksTableFile(childTableName, aggregatedFile, tableFileMap, memberCode, rowCounter); + aggregateBlocksTableFile(childTableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter); } while ((childLine = childReader.nextLine()) != null) { - childAccSerialNum = Long.parseLong(childLine.split(";")[3]); + childLineList = Arrays.asList(childLine.split(";")); + foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM"); + if (foundChildAccSerialNum.isEmpty()) { + throw new RuntimeException("ACC_SERIAL_NUM was not found!"); + } + childAccSerialNum = Long.parseLong(foundChildAccSerialNum.get()); if (childAccSerialNum.equals(ruUidSerialNum)) { - writer.write(childLine); - writer.newLine(); + aggregatedFileWriter.write(childLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); if (childTableName.equals("RU_COLLATERAL")) { - aggregateBlocksTableFile(childTableName, aggregatedFile, tableFileMap, memberCode, rowCounter); + aggregateBlocksTableFile(childTableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter); } } @@ -149,40 +167,52 @@ public class FileAggregatorCommon { } } - void aggregateBlocksTableFile(String parentTableName, Path aggregatedFile, Map tableFileMap, String memberCode, MutableLong rowCounter) { + void aggregateBlocksTableFile(String parentTableName, BufferedWriter aggregatedFileWriter, Map tableFileMap, String memberCode, MutableLong rowCounter) { try (BufferedReader parentFileReader = Files.newBufferedReader(tableFileMap.get(parentTableName)); - MovableBufferedReader blocksFileReader = new MovableBufferedReader(Files.newBufferedReader(tableFileMap.get(parentTableName + "_BLOCKS"))); - BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + MovableBufferedReader blocksFileReader = new MovableBufferedReader(Files.newBufferedReader(tableFileMap.get(parentTableName + "_BLOCKS")))) { String parentLine; while ((parentLine = parentFileReader.readLine()) != null) { - List lineList = Arrays.asList(parentLine.split(";")); - int memberCodeIndex = lineList.indexOf("MEMBER_CODE"); - if (memberCodeIndex != -1) { - if (!lineList.get(memberCodeIndex + 1).startsWith(memberCode)) { - continue; - } + List parentLineList = Arrays.asList(parentLine.split(";")); + Optional foundMemberCode = IndicSearchCommon.getRowValue(parentLineList, "MEMBER_CODE"); + + if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) { + continue; } - writer.write(parentLine); - writer.newLine(); + aggregatedFileWriter.write(parentLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); - Long parentSerialNum = Long.parseLong(parentLine.split(";")[2]); + Optional foundParentSerialNum = IndicSearchCommon.getRowValue(parentLineList, "SERIAL_NUM"); + if (foundParentSerialNum.isEmpty()) { + throw new RuntimeException("SERIAL_NUM was not found!"); + } + Long parentSerialNum = Long.parseLong(foundParentSerialNum.get()); String blocksLine = blocksFileReader.getCurrentLine(); if (blocksLine == null) { continue; } - Long blocksParentSerialNum = Long.parseLong(blocksLine.split(";")[4]); + List childLineList = Arrays.asList(blocksLine.split(";")); + Optional foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM"); + if (foundChildParentSerialNum.isEmpty()) { + throw new RuntimeException("PARENT_SERIAL_NUM was not found!"); + } + Long blocksParentSerialNum = Long.parseLong(foundChildParentSerialNum.get()); if (parentSerialNum > blocksParentSerialNum) { while ((blocksLine = blocksFileReader.nextLine()) != null) { - blocksParentSerialNum = Long.parseLong(blocksLine.split(";")[4]); + childLineList = Arrays.asList(blocksLine.split(";")); + foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM"); + if (foundChildParentSerialNum.isEmpty()) { + throw new RuntimeException("PARENT_SERIAL_NUM was not found!"); + } + blocksParentSerialNum = Long.parseLong(foundChildParentSerialNum.get()); if (blocksParentSerialNum.equals(parentSerialNum)) { - writer.write(blocksLine); - writer.newLine(); + aggregatedFileWriter.write(blocksLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); } @@ -191,16 +221,21 @@ public class FileAggregatorCommon { } } } else if (parentSerialNum.equals(blocksParentSerialNum)) { - writer.write(blocksLine); - writer.newLine(); + aggregatedFileWriter.write(blocksLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); while ((blocksLine = blocksFileReader.nextLine()) != null) { - blocksParentSerialNum = Long.parseLong(blocksLine.split(";")[4]); + childLineList = Arrays.asList(blocksLine.split(";")); + foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM"); + if (foundChildParentSerialNum.isEmpty()) { + throw new RuntimeException("PARENT_SERIAL_NUM was not found!"); + } + blocksParentSerialNum = Long.parseLong(foundChildParentSerialNum.get()); if (blocksParentSerialNum.equals(parentSerialNum)) { - writer.write(blocksLine); - writer.newLine(); + aggregatedFileWriter.write(blocksLine); + aggregatedFileWriter.newLine(); rowCounter.increment(); } diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidFileAggregator.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidFileAggregator.java index d0777d5..6fdf91b 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidFileAggregator.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidFileAggregator.java @@ -6,15 +6,16 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import org.apache.commons.lang3.mutable.MutableLong; import org.springframework.stereotype.Component; import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.util.IndicSearchCommon; @Component public class GutdfFidFileAggregator implements FileAggregator { @@ -57,37 +58,34 @@ public class GutdfFidFileAggregator implements FileAggregator { public Path aggregateFiles(Map tableFileMap, String memberCode, MutableLong rowCounter, UUID taskId) { Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("AGGREGATED-" + taskId + ".csv"); - for (String tableName : getRequiredTables()) { - if (tableName.equals("RU_UID")) { - fileAggregatorCommon.aggregateRuUidFiles(aggregatedFile, tableFileMap, memberCode, rowCounter); - } else if (tableName.equals("RU_BANKRUPTCY")) { - fileAggregatorCommon.aggregateBlocksTableFile(tableName, aggregatedFile, tableFileMap, memberCode, rowCounter); - } else { - try (BufferedReader reader = Files.newBufferedReader(tableFileMap.get(tableName)); - BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { - String line; + try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + for (String tableName : getRequiredTables()) { + if (tableName.equals("RU_UID")) { + fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter); + } else if (tableName.equals("RU_BANKRUPTCY")) { + fileAggregatorCommon.aggregateBlocksTableFile(tableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter); + } else { + try (BufferedReader reader = Files.newBufferedReader(tableFileMap.get(tableName))) { + String line; - while ((line = reader.readLine()) != null) { - List lineList = Arrays.asList(line.split(";")); - int memberCodeIndex = lineList.indexOf("MEMBER_CODE"); - if (memberCodeIndex != -1) { - if (!lineList.get(memberCodeIndex + 1).startsWith(memberCode)) { + while ((line = reader.readLine()) != null) { + Optional foundMemberCode = IndicSearchCommon.getRowValue(line, "MEMBER_CODE"); + + if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) { continue; } - } - writer.write(line); - writer.newLine(); - rowCounter.increment(); + aggregatedFileWriter.write(line); + aggregatedFileWriter.newLine(); + rowCounter.increment(); + } + } catch (IOException e) { + throw new RuntimeException(e); } - } catch (IOException e) { - throw new RuntimeException(e); } } - } - try (BufferedWriter writer = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { - writer.write("EOF"); + aggregatedFileWriter.write("EOF"); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/ru/nbch/indicsearchservice/util/ExcelWriter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java similarity index 87% rename from src/main/java/ru/nbch/indicsearchservice/util/ExcelWriter.java rename to src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java index b59cda5..49634f1 100644 --- a/src/main/java/ru/nbch/indicsearchservice/util/ExcelWriter.java +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java @@ -1,4 +1,4 @@ -package ru.nbch.indicsearchservice.util; +package ru.nbch.indicsearchservice.exporter; import jakarta.annotation.Nullable; import java.io.BufferedReader; @@ -8,12 +8,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.UUID; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.poi.ss.usermodel.BorderStyle; @@ -32,28 +31,17 @@ import org.springframework.stereotype.Component; import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; @Component -public class ExcelWriter { - private static final Logger log = LoggerFactory.getLogger(ExcelWriter.class); +public class ExcelExporter implements Exporter { + private static final Logger log = LoggerFactory.getLogger(ExcelExporter.class); private static final Long EXCEL_SHEET_MAX_SIZE = 1_048_575L; - private static final Set FIRST_LEVEL_TABLES = Set.of( - "RU_UID", - "RU_APPLICATION", - "RU_OBLIGPARTTAKE", - "RU_BANKRUPTCY", - "RU_SETTLED", - "RU_OTHERLEGAL", - "RU_CAPABILITY", - "RU_DEBT_BURDEN_INFO" - ); - private final ApplicationSettings settings; - public ExcelWriter(ApplicationSettings settings) { + public ExcelExporter(ApplicationSettings settings) { this.settings = settings; } - public void writeExcelFile(Path aggregatedFile, UUID taskId) { + public void writeExcelFile(Path aggregatedFile, UUID taskId, Long totalRows) { Path excelFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".xlsx"); try (OutputStream os = Files.newOutputStream(excelFile); @@ -67,16 +55,21 @@ public class ExcelWriter { MutableLong sheetRowCounter = new MutableLong(0L); MutableLong sheetTableCounter = new MutableLong(1L); + long count = 0L; + boolean isFirst = true; - String lastTableName = null; + String lastTableName; List rows = new ArrayList<>(); while (iterator.hasNext()) { String line = iterator.next(); + List lineList = Arrays.asList(line.split(";")); - if (!isFirst && (FIRST_LEVEL_TABLES.contains(line.split(";")[0]) || line.equals("EOF"))) { + log.info("Processing row: {} / {}", count++, totalRows); + + if (!isFirst && (FIRST_LEVEL_TABLES.contains(lineList.getFirst()) || line.equals("EOF"))) { lastTableName = rows.getFirst().split(";")[0]; writeToExcel(workbook, rows, sheetRowCounter, sheetTableCounter, headerCellStyle, cellStyle); - if (lastTableName != null && !lastTableName.equals(line.split(";")[0])) { + if (lastTableName != null && !lastTableName.equals(lineList.getFirst())) { sheetRowCounter.setValue(0L); sheetTableCounter.setValue(1L); } @@ -106,21 +99,26 @@ public class ExcelWriter { String parentTableName = rows.getFirst().split(";")[0]; Sheet tableSheet; - if (sheetRowCounter.longValue() + (rows.size() * 2L) >= EXCEL_SHEET_MAX_SIZE) { + int rowsApproximateSize = rows.stream().mapToInt(r -> r.split(";", -1).length).sum(); + if (sheetRowCounter.longValue() + (rowsApproximateSize * 2L) >= EXCEL_SHEET_MAX_SIZE) { tableSheet = createSheet(workbook, parentTableName.toUpperCase() + "_" + sheetTableCounter.incrementAndGet()); sheetRowCounter.setValue(0L); } else { - tableSheet = createSheet(workbook, parentTableName.toUpperCase() + "_" + sheetTableCounter.longValue()); + tableSheet = createSheet(workbook, parentTableName.toUpperCase()); } for (String row : rows) { List rowList = Arrays.asList(row.split(";", -1)); String tableName = rowList.getFirst(); - List rowSublist = rowList.subList(5, rowList.size()); - Map rowValues = new HashMap<>(); + List rowSublist = rowList.subList(1, rowList.size()); + Map rowValues = new LinkedHashMap<>(); for (int i = 0; i + 1 <= rowSublist.size(); i = i + 2) { - rowValues.put(rowSublist.get(i), rowSublist.get(i + 1)); + try { + rowValues.put(rowSublist.get(i), rowSublist.get(i + 1)); + } catch (Exception e) { + throw new RuntimeException(e); + } } createHeaderRow( diff --git a/src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java new file mode 100644 index 0000000..ae0a177 --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java @@ -0,0 +1,16 @@ +package ru.nbch.indicsearchservice.exporter; + +import java.util.Set; + +public interface Exporter { + Set FIRST_LEVEL_TABLES = Set.of( + "RU_UID", + "RU_APPLICATION", + "RU_OBLIGPARTTAKE", + "RU_BANKRUPTCY", + "RU_SETTLED", + "RU_OTHERLEGAL", + "RU_CAPABILITY", + "RU_DEBT_BURDEN_INFO" + ); +} diff --git a/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java new file mode 100644 index 0000000..8776d12 --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java @@ -0,0 +1,113 @@ +package ru.nbch.indicsearchservice.exporter; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; +import ru.nbch.indicsearchservice.model.TableData; +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.json.JsonMapper; + +@Component +public class JsonExporter implements Exporter { + private static final Logger log = LoggerFactory.getLogger(JsonExporter.class); + + private final ApplicationSettings settings; + private final JsonMapper jsonMapper; + + public JsonExporter(ApplicationSettings settings, + JsonMapper jsonMapper) { + this.settings = settings; + this.jsonMapper = jsonMapper; + } + + public void writeJsonFile(Path aggregatedFile, UUID taskId) { + Path jsonFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".json"); + + try (BufferedReader reader = Files.newBufferedReader(aggregatedFile); + JsonGenerator jsonGenerator = jsonMapper.createGenerator(Files.newOutputStream(jsonFile))) { + jsonGenerator.writeStartArray(); + + Iterator iterator = reader.lines().iterator(); + List rows = new ArrayList<>(); + boolean isFirst = true; + + while (iterator.hasNext()) { + String line = iterator.next(); + + if (line == null || line.trim().isEmpty()) { + continue; + } + + List lineList = Arrays.asList(line.split(";", -1)); + String tableName = lineList.getFirst(); + + if (!isFirst && (FIRST_LEVEL_TABLES.contains(tableName) || line.equals("EOF"))) { + TableData parentData = buildTableDataHierarchy(rows); + if (parentData != null) { + jsonMapper.writeValue(jsonGenerator, parentData); + } + rows.clear(); + } + + rows.add(line); + isFirst = false; + } + + jsonGenerator.writeEndArray(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + + private TableData buildTableDataHierarchy(List rows) { + if (rows == null || rows.isEmpty()) { + return null; + } + + TableData parentTableData = parseSingleRow(rows.getFirst()); + if (parentTableData == null) { + return null; + } + + for (int i = 1; i < rows.size(); i++) { + TableData childTableData = parseSingleRow(rows.get(i)); + if (childTableData != null) { + parentTableData.getRelatedTableData().add(childTableData); + } + } + + return parentTableData; + } + + private TableData parseSingleRow(String row) { + String[] parts = row.split(";", -1); + + if (parts.length == 0 || parts[0].isEmpty()) { + return null; + } + + String tableName = parts[0]; + List fieldData = new ArrayList<>(); + + for (int i = 1; i + 1 < parts.length; i += 2) { + String columnName = parts[i]; + String columnValue = parts[i + 1]; + + if (columnName != null && !columnName.isEmpty()) { + fieldData.add(new TableData.FieldData(columnName, columnValue)); + } + } + + return new TableData(tableName, fieldData); + } +} diff --git a/src/main/java/ru/nbch/indicsearchservice/model/TableData.java b/src/main/java/ru/nbch/indicsearchservice/model/TableData.java index 859421e..8735464 100644 --- a/src/main/java/ru/nbch/indicsearchservice/model/TableData.java +++ b/src/main/java/ru/nbch/indicsearchservice/model/TableData.java @@ -1,9 +1,11 @@ package ru.nbch.indicsearchservice.model; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.ArrayList; import java.util.List; import java.util.Objects; +@JsonPropertyOrder({"tableName", "fieldData", "relatedTableData"}) public class TableData { private String tableName; private List fieldData = new ArrayList<>(); diff --git a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java index fc6b6a3..5ba567d 100644 --- a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java +++ b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java @@ -19,34 +19,33 @@ import ru.nbch.indicsearchservice.aggregator.FileAggregator; import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto; import ru.nbch.indicsearchservice.dto.response.AddSearchJobResponseDto; import ru.nbch.indicsearchservice.dto.response.JobStatusResponseDto; +import ru.nbch.indicsearchservice.exporter.ExcelExporter; +import ru.nbch.indicsearchservice.exporter.JsonExporter; import ru.nbch.indicsearchservice.loader.TableLoader; -import ru.nbch.indicsearchservice.util.ExcelWriter; import ru.nbch.indicsearchservice.util.IndicSearchUtils; -import tools.jackson.databind.json.JsonMapper; @Service public class IndicSearchService { private static final Logger log = LoggerFactory.getLogger(IndicSearchService.class); - private static final boolean IS_EXPORT_AS_EXCEL = true; + private static final boolean IS_EXPORT_AS_EXCEL = false; private final List tableLoaders; private final List fileAggregators; private final IndicSearchUtils indicSearchUtils; - private final ExcelWriter excelWriter; - - private final JsonMapper jsonMapper; + private final ExcelExporter excelExporter; + private final JsonExporter jsonExporter; public IndicSearchService(List tableLoaders, List fileAggregators, IndicSearchUtils indicSearchUtils, - ExcelWriter excelWriter, - JsonMapper jsonMapper) { + ExcelExporter excelExporter, + JsonExporter jsonExporter) { this.tableLoaders = tableLoaders; this.fileAggregators = fileAggregators; this.indicSearchUtils = indicSearchUtils; - this.excelWriter = excelWriter; - this.jsonMapper = jsonMapper; + this.excelExporter = excelExporter; + this.jsonExporter = jsonExporter; } public AddSearchJobResponseDto addSearchJob(AddSearchJobRequestDto request) { @@ -112,13 +111,9 @@ public class IndicSearchService { sw.start(); if (!IS_EXPORT_AS_EXCEL) { -// try { -// Files.write(Path.of("1.json"), jsonMapper.writeValueAsBytes(result)); -// } catch (IOException e) { -// throw new RuntimeException(e); -// } + jsonExporter.writeJsonFile(aggregatedFile, taskId); } else { - excelWriter.writeExcelFile(aggregatedFile, taskId); + excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue()); } sw.stop(); diff --git a/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java b/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java new file mode 100644 index 0000000..b0beaf0 --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java @@ -0,0 +1,19 @@ +package ru.nbch.indicsearchservice.util; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +public class IndicSearchCommon { + public static Optional getRowValue(String string, String key) { + return getRowValue(Arrays.asList(string.split(";", -1)), key); + } + + public static Optional getRowValue(List stringValues, String key) { + int keyIndex = stringValues.indexOf(key); + if (keyIndex != -1) { + return Optional.of(stringValues.get(keyIndex + 1)); + } + return Optional.empty(); + } +} diff --git a/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchUtils.java b/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchUtils.java index b9e3880..f3934e6 100644 --- a/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchUtils.java +++ b/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchUtils.java @@ -5,14 +5,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.sql.Date; -import java.sql.Timestamp; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -20,7 +14,6 @@ import java.util.function.Function; import org.apache.ibatis.cursor.Cursor; import org.springframework.stereotype.Component; import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; -import ru.nbch.indicsearchservice.model.TableData; import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.SearchParameter; @@ -44,30 +37,6 @@ public class IndicSearchUtils { StringBuilder sb = new StringBuilder(); sb.append(tableName).append(";"); - if (row.containsKey("FID") && row.get("FID") != null) { - sb.append(row.get("FID").toString()).append(";"); - } else { - sb.append(";"); - } - - if (row.containsKey("SERIAL_NUM") && row.get("SERIAL_NUM") != null) { - sb.append(row.get("SERIAL_NUM").toString()).append(";"); - } else { - sb.append(";"); - } - - if (row.containsKey("ACC_SERIAL_NUM") && row.get("ACC_SERIAL_NUM") != null) { - sb.append(row.get("ACC_SERIAL_NUM").toString()).append(";"); - } else { - sb.append(";"); - } - - if (row.containsKey("PARENT_SERIAL_NUM") && row.get("PARENT_SERIAL_NUM") != null) { - sb.append(row.get("PARENT_SERIAL_NUM").toString()).append(";"); - } else { - sb.append(";"); - } - for (Map.Entry entry : row.entrySet()) { String columnName = entry.getKey(); Object columnValue = entry.getValue(); @@ -111,51 +80,4 @@ public class IndicSearchUtils { return fids; } - - public TableData mapToTableData(String table, Map row) { - List fields = row.entrySet().stream() - .map(e -> new TableData.FieldData(e.getKey(), parseTableValue(e.getValue()))) - .toList(); - - return new TableData(table, fields); - } - - private Object parseTableValue(Object value) { - return switch (value) { - case Timestamp timestamp -> - timestamp.toLocalDateTime().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); - case Date date -> { - LocalDate localDate = date.toLocalDate(); - if (localDate.equals(Date.valueOf("0001-01-01").toLocalDate()) || - localDate.equals(Date.valueOf("0001-01-03").toLocalDate())) { - yield null; - } - yield localDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); - } - case Integer i -> { - if (i == Integer.MAX_VALUE) { - yield null; - } - yield i; - } - case Long l -> { - if (l == Long.MAX_VALUE) { - yield null; - } - yield l; - } - case null -> null; - default -> value; - }; - } - - public List getFirstLevelTables(Map> tableDataMapByFid) { - List result = new ArrayList<>(); - - for (Map.Entry> entry : tableDataMapByFid.entrySet()) { - if (entry.getKey().equals("RU_UID")) {} - } - - return result; - } }