From 61d88923181d888cc4c8da0054ed700287e87169 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 28 May 2026 13:30:44 +0300 Subject: [PATCH] JSON exporter fixed, JsonExporterTest added --- .../indicsearchservice/exporter/Exporter.java | 7 +- .../exporter/JsonExporter.java | 50 +++++++++++-- .../service/IndicSearchService.java | 2 +- .../exporter/JsonExporterTest.java | 75 +++++++++++++++++++ src/test/resources/application.properties | 18 +++++ .../storage/test-json-aggregated.csv | 6 ++ 6 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java create mode 100644 src/test/resources/application.properties create mode 100644 src/test/resources/storage/test-json-aggregated.csv diff --git a/src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java index ae0a177..f52f9f7 100644 --- a/src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java @@ -11,6 +11,11 @@ public interface Exporter { "RU_SETTLED", "RU_OTHERLEGAL", "RU_CAPABILITY", - "RU_DEBT_BURDEN_INFO" + "RU_DEBT_BURDEN_INFO", + "ACCOUNT", + "INFORMATION", + "BANKRUPTCY", + "CONSUMER_BANKRUPTCY", + "LEGAL" ); } diff --git a/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java index 34a7360..af66908 100644 --- a/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +21,7 @@ import org.springframework.stereotype.Component; import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; import ru.nbch.indicsearchservice.model.TableData; import ru.nbch.indicsearchservice.model.TableMetaData; +import ru.nbch.indicsearchservice.model.enums.SearchParameter; import tools.jackson.core.JsonGenerator; import tools.jackson.databind.json.JsonMapper; @@ -36,11 +38,18 @@ public class JsonExporter implements Exporter { this.jsonMapper = jsonMapper; } - public void writeJsonFile(@Nullable Path aggregatedFile, UUID taskId, Map tableMetaDataMap) { + public void writeJsonFile(@Nullable Path aggregatedFile, + UUID taskId, + SearchParameter searchParameter, + Map tableMetaDataMap) { if (aggregatedFile == null) { return; } + boolean isPassportOgrnFid = searchParameter == SearchParameter.PASSPORT || + searchParameter == SearchParameter.OGRN || + searchParameter == SearchParameter.FID; + Path jsonFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".json"); try (BufferedReader reader = Files.newBufferedReader(aggregatedFile); @@ -48,12 +57,14 @@ public class JsonExporter implements Exporter { jsonGenerator.writeStartObject(); jsonGenerator.writeName("datasheets"); jsonGenerator.writeStartArray(); - jsonGenerator.writeStartArray(); Iterator iterator = reader.lines().iterator(); List rows = new ArrayList<>(); boolean isFirst = true; + boolean isFirstArray = true; + boolean isFileEmpty = true; + AtomicReference previousFid = new AtomicReference<>(); while (iterator.hasNext()) { String line = iterator.next(); @@ -67,7 +78,30 @@ public class JsonExporter implements Exporter { if (!isFirst && (FIRST_LEVEL_TABLES.contains(tableName) || line.equals("EOF"))) { TableData parentData = buildTableDataHierarchy(rows, tableMetaDataMap); if (parentData != null) { - jsonMapper.writeValue(jsonGenerator, parentData); + if (isPassportOgrnFid) { + Object currentFidField = parentData.getFieldData().stream() + .filter(fd -> fd.columnName().equals("FID") && fd.columnValue() != null) + .findFirst() + .orElse(new TableData.FieldData("FID", null)) + .columnValue(); + + if (previousFid.get() == null || (currentFidField != null && Integer.parseInt(currentFidField.toString()) != previousFid.get())) { + previousFid.set(Integer.parseInt(currentFidField.toString())); + if (!isFirstArray) { + jsonGenerator.writeEndArray(); + } + jsonGenerator.writeStartArray(); + jsonMapper.writeValue(jsonGenerator, parentData); + isFirstArray = false; + isFileEmpty = false; + } else { + jsonMapper.writeValue(jsonGenerator, parentData); + } + } else { + jsonGenerator.writeStartArray(); + jsonMapper.writeValue(jsonGenerator, parentData); + jsonGenerator.writeEndArray(); + } } rows.clear(); } @@ -76,8 +110,14 @@ public class JsonExporter implements Exporter { isFirst = false; } - jsonGenerator.writeEndArray(); - jsonGenerator.writeEndArray(); + if (!isPassportOgrnFid) { + jsonGenerator.writeEndArray(); // Закрываем массив объекта + } else if (!isFileEmpty) { + jsonGenerator.writeEndArray(); // Закрываем массивы объектов с одним FID для PASSPORT, OGRN, FID + jsonGenerator.writeEndArray(); // Закрываем общий массив + } else { + jsonGenerator.writeEndArray(); // Если файл пуст, закрываем только общий массив + } jsonGenerator.writeEndObject(); } catch (IOException e) { log.error(e.getMessage(), e); diff --git a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java index 5c5ddf1..de35147 100644 --- a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java +++ b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java @@ -143,7 +143,7 @@ public class IndicSearchService { sw.start(); if (rowCounter.longValue() < rowLimit && !IS_EXPORT_AS_EXCEL) { - jsonExporter.writeJsonFile(aggregatedFile, taskId, tableMetaDataMap); + jsonExporter.writeJsonFile(aggregatedFile, taskId, request.getSearchParameter(), tableMetaDataMap); } else { fillTableMetaDataWithLocalization(tableMetaDataMap); excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue(), tableMetaDataMap); diff --git a/src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java b/src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java new file mode 100644 index 0000000..f2562b2 --- /dev/null +++ b/src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java @@ -0,0 +1,75 @@ +package ru.nbch.indicsearchservice.exporter; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import ru.nbch.indicsearchservice.config.settings.ApplicationSettings; +import ru.nbch.indicsearchservice.model.TableMetaData; +import ru.nbch.indicsearchservice.model.enums.SearchParameter; + +@SpringBootTest +class JsonExporterTest { + @Autowired + ApplicationSettings applicationSettings; + @Autowired + JsonExporter jsonExporter; + + @Test + void writeJsonFile_shouldCreateJsonFilePASSPORT() throws IOException { + Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv"); + UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907"); + + TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID"); + ruUidTableTableMetaData.addField("FID"); + ruUidTableTableMetaData.addField("SERIAL_NUM"); + TableMetaData ruApplicationTableTableMetaData = new TableMetaData("RU_APPLICATION"); + ruApplicationTableTableMetaData.addField("FID"); + ruApplicationTableTableMetaData.addField("SERIAL_NUM"); + + Map tableMetaDataMap = new HashMap<>(); + tableMetaDataMap.put("RU_UID", ruUidTableTableMetaData); + tableMetaDataMap.put("RU_APPLICATION", ruApplicationTableTableMetaData); + + jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.PASSPORT, tableMetaDataMap); + + Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json"); + List resultFileStrings = Files.readAllLines(resultFile); + + assertThat(resultFileStrings.getFirst()) + .isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]},{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}"); + } + + @Test + void writeJsonFile_shouldCreateJsonFileUUID() throws IOException { + Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv"); + UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907"); + + TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID"); + ruUidTableTableMetaData.addField("FID"); + ruUidTableTableMetaData.addField("SERIAL_NUM"); + TableMetaData ruApplicationTableTableMetaData = new TableMetaData("RU_APPLICATION"); + ruApplicationTableTableMetaData.addField("FID"); + ruApplicationTableTableMetaData.addField("SERIAL_NUM"); + + Map tableMetaDataMap = new HashMap<>(); + tableMetaDataMap.put("RU_UID", ruUidTableTableMetaData); + tableMetaDataMap.put("RU_APPLICATION", ruApplicationTableTableMetaData); + + jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.UUID, tableMetaDataMap); + + Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json"); + List resultFileStrings = Files.readAllLines(resultFile); + + assertThat(resultFileStrings.getFirst()) + .isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]}],[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}"); + } +} \ No newline at end of file diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties new file mode 100644 index 0000000..8b1900f --- /dev/null +++ b/src/test/resources/application.properties @@ -0,0 +1,18 @@ +spring.application.name=indic-search-service + +logging.level.org.springframework.boot.context.config=DEBUG + +# Database properties +app.data-source.driver=com.ibm.db2.jcc.DB2Driver +app.data-source.username=ros9lk +app.data-source.password=ros9lk +app.data-source.url=jdbc:db2://10.230.227.21:2667/DEVSUPPL:currentSchema=ROS9LK; +app.data-source.connection-timeout=40000 +app.data-source.validation-timeout=5000 +app.data-source.idle-timeout=600000 +app.data-source.max-pool-size=10 +app.data-source.max-lifetime=1800000 +app.data-source.keep-alive-time=120000 + +app.storage-directory=C:\\Users\\inikolaev\\Desktop\\1 +#app.storage-directory=C:\\Users\\ivan\\Desktop\\storage \ No newline at end of file diff --git a/src/test/resources/storage/test-json-aggregated.csv b/src/test/resources/storage/test-json-aggregated.csv new file mode 100644 index 0000000..51fb959 --- /dev/null +++ b/src/test/resources/storage/test-json-aggregated.csv @@ -0,0 +1,6 @@ +RU_UID%%%FID%%%1%%%SERIAL_NUM%%%11%%% +RU_UID%%%FID%%%1%%%SERIAL_NUM%%%21%%% +RU_TRADE%%%FID%%%1%%%SERIAL_NUM%%%31%%%ACC_SERIAL_NUM%%%21%%% +RU_APPLICATION%%%FID%%%2%%%SERIAL_NUM%%%12%%% +RU_APPLICATION%%%FID%%%3%%%SERIAL_NUM%%%22%%% +EOF \ No newline at end of file