treshenkov/indic-search-service
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

JSON exporter fixed, JsonExporterTest added

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2026-05-28 13:30:44 +03:00
parent e81c5e1dcf
commit 61d8892318
6 changed files with 151 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/main/java/ru/nbch/indicsearchservice/exporter/Exporter.java
View file

@ -11,6 +11,11 @@ public interface Exporter {
"RU_SETTLED",
"RU_OTHERLEGAL",
"RU_CAPABILITY",
"RU_DEBT_BURDEN_INFO"
"RU_DEBT_BURDEN_INFO",
"ACCOUNT",
"INFORMATION",
"BANKRUPTCY",
"CONSUMER_BANKRUPTCY",
"LEGAL"
);
}

50
src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java
View file

@ -13,6 +13,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -20,6 +21,7 @@ import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.config.settings.ApplicationSettings;
import ru.nbch.indicsearchservice.model.TableData;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import tools.jackson.core.JsonGenerator;
import tools.jackson.databind.json.JsonMapper;
@ -36,11 +38,18 @@ public class JsonExporter implements Exporter {
this.jsonMapper = jsonMapper;
}
public void writeJsonFile(@Nullable Path aggregatedFile, UUID taskId, Map<String, TableMetaData> tableMetaDataMap) {
public void writeJsonFile(@Nullable Path aggregatedFile,
UUID taskId,
SearchParameter searchParameter,
Map<String, TableMetaData> tableMetaDataMap) {
if (aggregatedFile == null) {
return;
}
boolean isPassportOgrnFid = searchParameter == SearchParameter.PASSPORT ||
searchParameter == SearchParameter.OGRN ||
searchParameter == SearchParameter.FID;
Path jsonFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".json");
try (BufferedReader reader = Files.newBufferedReader(aggregatedFile);
@ -48,12 +57,14 @@ public class JsonExporter implements Exporter {
jsonGenerator.writeStartObject();
jsonGenerator.writeName("datasheets");
jsonGenerator.writeStartArray();
jsonGenerator.writeStartArray();
Iterator<String> iterator = reader.lines().iterator();
List<String> rows = new ArrayList<>();
boolean isFirst = true;
boolean isFirstArray = true;
boolean isFileEmpty = true;
AtomicReference<Integer> previousFid = new AtomicReference<>();
while (iterator.hasNext()) {
String line = iterator.next();
@ -67,7 +78,30 @@ public class JsonExporter implements Exporter {
if (!isFirst && (FIRST_LEVEL_TABLES.contains(tableName) || line.equals("EOF"))) {
TableData parentData = buildTableDataHierarchy(rows, tableMetaDataMap);
if (parentData != null) {
jsonMapper.writeValue(jsonGenerator, parentData);
if (isPassportOgrnFid) {
Object currentFidField = parentData.getFieldData().stream()
.filter(fd -> fd.columnName().equals("FID") && fd.columnValue() != null)
.findFirst()
.orElse(new TableData.FieldData("FID", null))
.columnValue();
if (previousFid.get() == null || (currentFidField != null && Integer.parseInt(currentFidField.toString()) != previousFid.get())) {
previousFid.set(Integer.parseInt(currentFidField.toString()));
if (!isFirstArray) {
jsonGenerator.writeEndArray();
}
jsonGenerator.writeStartArray();
jsonMapper.writeValue(jsonGenerator, parentData);
isFirstArray = false;
isFileEmpty = false;
} else {
jsonMapper.writeValue(jsonGenerator, parentData);
}
} else {
jsonGenerator.writeStartArray();
jsonMapper.writeValue(jsonGenerator, parentData);
jsonGenerator.writeEndArray();
}
}
rows.clear();
}
@ -76,8 +110,14 @@ public class JsonExporter implements Exporter {
isFirst = false;
}
jsonGenerator.writeEndArray();
jsonGenerator.writeEndArray();
if (!isPassportOgrnFid) {
jsonGenerator.writeEndArray(); // Закрываем массив объекта
} else if (!isFileEmpty) {
jsonGenerator.writeEndArray(); // Закрываем массивы объектов с одним FID для PASSPORT, OGRN, FID
jsonGenerator.writeEndArray(); // Закрываем общий массив
} else {
jsonGenerator.writeEndArray(); // Если файл пуст, закрываем только общий массив
}
jsonGenerator.writeEndObject();
} catch (IOException e) {
log.error(e.getMessage(), e);

2
src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java
View file

@ -143,7 +143,7 @@ public class IndicSearchService {
sw.start();
if (rowCounter.longValue() < rowLimit && !IS_EXPORT_AS_EXCEL) {
jsonExporter.writeJsonFile(aggregatedFile, taskId, tableMetaDataMap);
jsonExporter.writeJsonFile(aggregatedFile, taskId, request.getSearchParameter(), tableMetaDataMap);
} else {
fillTableMetaDataWithLocalization(tableMetaDataMap);
excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue(), tableMetaDataMap);

75
src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java Normal file
View file

@ -0,0 +1,75 @@
package ru.nbch.indicsearchservice.exporter;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import ru.nbch.indicsearchservice.config.settings.ApplicationSettings;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
@SpringBootTest
class JsonExporterTest {
@Autowired
ApplicationSettings applicationSettings;
@Autowired
JsonExporter jsonExporter;
@Test
void writeJsonFile_shouldCreateJsonFilePASSPORT() throws IOException {
Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv");
UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907");
TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID");
ruUidTableTableMetaData.addField("FID");
ruUidTableTableMetaData.addField("SERIAL_NUM");
TableMetaData ruApplicationTableTableMetaData = new TableMetaData("RU_APPLICATION");
ruApplicationTableTableMetaData.addField("FID");
ruApplicationTableTableMetaData.addField("SERIAL_NUM");
Map<String, TableMetaData> tableMetaDataMap = new HashMap<>();
tableMetaDataMap.put("RU_UID", ruUidTableTableMetaData);
tableMetaDataMap.put("RU_APPLICATION", ruApplicationTableTableMetaData);
jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.PASSPORT, tableMetaDataMap);
Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json");
List<String> resultFileStrings = Files.readAllLines(resultFile);
assertThat(resultFileStrings.getFirst())
.isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]},{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}");
}
@Test
void writeJsonFile_shouldCreateJsonFileUUID() throws IOException {
Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv");
UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907");
TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID");
ruUidTableTableMetaData.addField("FID");
ruUidTableTableMetaData.addField("SERIAL_NUM");
TableMetaData ruApplicationTableTableMetaData = new TableMetaData("RU_APPLICATION");
ruApplicationTableTableMetaData.addField("FID");
ruApplicationTableTableMetaData.addField("SERIAL_NUM");
Map<String, TableMetaData> tableMetaDataMap = new HashMap<>();
tableMetaDataMap.put("RU_UID", ruUidTableTableMetaData);
tableMetaDataMap.put("RU_APPLICATION", ruApplicationTableTableMetaData);
jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.UUID, tableMetaDataMap);
Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json");
List<String> resultFileStrings = Files.readAllLines(resultFile);
assertThat(resultFileStrings.getFirst())
.isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]}],[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}");
}
}

18
src/test/resources/application.properties Normal file
View file

@ -0,0 +1,18 @@
spring.application.name=indic-search-service
logging.level.org.springframework.boot.context.config=DEBUG
# Database properties
app.data-source.driver=com.ibm.db2.jcc.DB2Driver
app.data-source.username=ros9lk
app.data-source.password=ros9lk
app.data-source.url=jdbc:db2://10.230.227.21:2667/DEVSUPPL:currentSchema=ROS9LK;
app.data-source.connection-timeout=40000
app.data-source.validation-timeout=5000
app.data-source.idle-timeout=600000
app.data-source.max-pool-size=10
app.data-source.max-lifetime=1800000
app.data-source.keep-alive-time=120000
app.storage-directory=C:\\Users\\inikolaev\\Desktop\\1
#app.storage-directory=C:\\Users\\ivan\\Desktop\\storage

6
src/test/resources/storage/test-json-aggregated.csv Normal file
View file

@ -0,0 +1,6 @@
RU_UID%%%FID%%%1%%%SERIAL_NUM%%%11%%%
RU_UID%%%FID%%%1%%%SERIAL_NUM%%%21%%%
RU_TRADE%%%FID%%%1%%%SERIAL_NUM%%%31%%%ACC_SERIAL_NUM%%%21%%%
RU_APPLICATION%%%FID%%%2%%%SERIAL_NUM%%%12%%%
RU_APPLICATION%%%FID%%%3%%%SERIAL_NUM%%%22%%%
EOF
1 RU_UID%%%FID%%%1%%%SERIAL_NUM%%%11%%%
2 RU_UID%%%FID%%%1%%%SERIAL_NUM%%%21%%%
3 RU_TRADE%%%FID%%%1%%%SERIAL_NUM%%%31%%%ACC_SERIAL_NUM%%%21%%%
4 RU_APPLICATION%%%FID%%%2%%%SERIAL_NUM%%%12%%%
5 RU_APPLICATION%%%FID%%%3%%%SERIAL_NUM%%%22%%%
6 EOF