treshenkov/indic-search-service
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

INDIC search fixed

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2026-05-28 16:38:47 +03:00
parent 61d8892318
commit cb2b3dddef
20 changed files with 507 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java
View file

@ -63,8 +63,6 @@ public class FileAggregatorCommon {
void aggregateRuUidFiles(BufferedWriter aggregatedFileWriter,
Map<String, Path> tableFileMap,
String memberCode,
String acctNum,
String uuid,
MutableLong rowCounter) {
try (BufferedReader ruUidFileReader = Files.newBufferedReader(tableFileMap.get("RU_UID"))) {
Map<String, MovableBufferedReader> childTablesReadersMap = new LinkedHashMap<>();
@ -88,21 +86,6 @@ public class FileAggregatorCommon {
continue;
}
if (acctNum != null) {
Optional<String> foundAcctNum = IndicSearchCommon.getRowValue(lineList, "ACCT_NUM");
if (foundAcctNum.isPresent() && !foundAcctNum.get().equals(acctNum)) {
continue;
}
}
if (uuid != null) {
Optional<String> foundUuid = IndicSearchCommon.getRowValue(lineList, "UUID");
if (foundUuid.isPresent() && !foundUuid.get().equals(uuid)) {
continue;
}
}
aggregatedFileWriter.write(ruUidLine);
aggregatedFileWriter.newLine();
rowCounter.increment();
@ -281,9 +264,6 @@ public class FileAggregatorCommon {
void aggregateAccountFiles(BufferedWriter aggregatedFileWriter,
Map<String, Path> tableFileMap,
String memberCode,
String acctNum,
String uuid,
String serialNum,
MutableLong rowCounter) {
try (BufferedReader accountFileReader = Files.newBufferedReader(tableFileMap.get("ACCOUNT"))) {
Map<String, MovableBufferedReader> childTablesReadersMap = new LinkedHashMap<>();
@ -307,32 +287,11 @@ public class FileAggregatorCommon {
continue;
}
if (acctNum != null) {
Optional<String> foundAcctNum = IndicSearchCommon.getRowValue(lineList, "ACCT_NUM");
if (foundAcctNum.isPresent() && !foundAcctNum.get().equals(acctNum)) {
continue;
}
}
if (uuid != null) {
Optional<String> foundUuid = IndicSearchCommon.getRowValue(lineList, "UUID");
if (foundUuid.isPresent() && !foundUuid.get().equals(uuid)) {
continue;
}
}
Optional<String> foundAccountSerialNum = IndicSearchCommon.getRowValue(lineList, "SERIAL_NUM");
if (serialNum != null) {
if (foundAccountSerialNum.isPresent() && !foundAccountSerialNum.get().equals(serialNum)) {
continue;
}
}
aggregatedFileWriter.write(accountLine);
aggregatedFileWriter.newLine();
rowCounter.increment();
Optional<String> foundAccountSerialNum = IndicSearchCommon.getRowValue(lineList, "SERIAL_NUM");
if (foundAccountSerialNum.isEmpty()) {
throw new InternalException("SERIAL_NUM was not found in line " + accountLine);
}

2
src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidPassportOgrnFileAggregator.java
View file

@ -55,7 +55,7 @@ public class GutdfFidPassportOgrnFileAggregator implements FileAggregator {
try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
for (String tableName : REQUIRED_TABLES) {
if (tableName.equals("RU_UID")) {
fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, null, null, rowCounter);
fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
} else if (tableName.equals("RU_BANKRUPTCY")) {
fileAggregatorCommon.aggregateBlocksTableFile(tableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
} else {

17
src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuApplicationFileAggregator.java
View file

@ -42,23 +42,6 @@ public class GutdfRuApplicationFileAggregator implements FileAggregator {
continue;
}
if (searchParameter == SearchParameter.APPLICATION_NUMBER && searchValue != null) {
searchValue = searchValue.length() > 38 ? searchValue.substring(0, 38) : searchValue;
Optional<String> foundApplicationNumber = IndicSearchCommon.getRowValue(line, "APPLICATION_NUMBER");
if (foundApplicationNumber.isPresent() && !foundApplicationNumber.get().equals(searchValue)) {
continue;
}
}
if (searchParameter == SearchParameter.OLD_APP_NUM && searchValue != null) {
Optional<String> foundOldAppNum = IndicSearchCommon.getRowValue(line, "OLD_APP_NUM");
if (foundOldAppNum.isPresent() && !foundOldAppNum.get().equals(searchValue)) {
continue;
}
}
aggregatedFileWriter.write(line);
aggregatedFileWriter.newLine();
rowCounter.increment();

6
src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuUidFileAggregator.java
View file

@ -32,11 +32,7 @@ public class GutdfRuUidFileAggregator implements FileAggregator {
Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("AGGREGATED-" + taskId + ".csv");
try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
if (searchParameter == SearchParameter.UUID) {
fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, null, searchValue, rowCounter);
} else {
fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, searchValue, null, rowCounter);
}
fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
aggregatedFileWriter.write("EOF");
} catch (IOException e) {

8
src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfAccountFileAggregator.java
View file

@ -32,13 +32,7 @@ public class TutdfAccountFileAggregator implements FileAggregator {
Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("AGGREGATED-" + taskId + ".csv");
try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
if (searchParameter == SearchParameter.UUID) {
fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, null, searchValue, null, rowCounter);
} else if (searchParameter == SearchParameter.ACCT_NUM) {
fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, searchValue, null, null, rowCounter);
} else {
fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, null, null, searchValue, rowCounter);
}
fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
aggregatedFileWriter.write("EOF");
} catch (IOException e) {

2
src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfFidPassportOgrnFileAggregator.java
View file

@ -52,7 +52,7 @@ public class TutdfFidPassportOgrnFileAggregator implements FileAggregator {
try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
for (String tableName : REQUIRED_TABLES) {
if (tableName.equals("ACCOUNT")) {
fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, null, null, null, rowCounter);
fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter);
} else {
try (BufferedReader reader = Files.newBufferedReader(tableFileMap.get(tableName))) {
String line;

15
src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfInformationFileAggregator.java
View file

@ -42,21 +42,6 @@ public class TutdfInformationFileAggregator implements FileAggregator {
continue;
}
if (searchParameter == SearchParameter.OLD_APP_NUM && searchValue != null) {
Optional<String> foundOldAppNum = IndicSearchCommon.getRowValue(line, "APP_NUM");
if (foundOldAppNum.isPresent() && !foundOldAppNum.get().equals(searchValue)) {
continue;
}
}
if (searchParameter == SearchParameter.SERIAL_NUM_APP && searchValue != null) {
Optional<String> foundSerialNum = IndicSearchCommon.getRowValue(line, "SERIAL_NUM");
if (foundSerialNum.isPresent() && !foundSerialNum.get().equals(searchValue)) {
continue;
}
}
aggregatedFileWriter.write(line);
aggregatedFileWriter.newLine();
rowCounter.increment();

2
src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java
View file

@ -137,7 +137,7 @@ public class ExcelExporter implements Exporter {
String columnValue = rowSublist.get(i + 1);
if (columnName != null && allowedFields.contains(columnName.toUpperCase())) {
rowValues.put(columnName, Pair.of(rowMetaData.getFieldMetaData().get(columnName).fieldLocalization(), columnValue));
rowValues.put(columnName, Pair.of(rowMetaData.getFieldMetaData().get(columnName).getFieldLocalization(), columnValue));
}
}

33
src/main/java/ru/nbch/indicsearchservice/loader/AccountTableLoader.java
View file

@ -15,6 +15,7 @@ import ru.nbch.indicsearchservice.loader.subtable.AccountSubTableLoader;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.util.IndicSearchCommon;
import ru.nbch.indicsearchservice.util.IndicSearchUtils;
import ru.nbch.indicsearchservice.util.MapperHolder;
@ -66,8 +67,36 @@ public class AccountTableLoader implements FirstLevelTableLoader, TableLoaderFor
Map<String, Path> tableFileMap,
Map<String, TableMetaData> tableMetaDataMap) {
if (dataType == DataType.TUTDF) {
if (searchParameter == SearchParameter.UUID || searchParameter == SearchParameter.ACCT_NUM || searchParameter == SearchParameter.SERIAL_NUM_ACCT) {
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromAccount(memberCode)) {
if (searchParameter == SearchParameter.UUID) {
if (IndicSearchCommon.isCastableField(getTableName(), "UUID")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("UUID").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByUuidAndMemberCodeFromAccount(searchValue, memberCode)) {
Set<Integer> foundFids = new HashSet<>();
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap));
accountSubTableLoaders.parallelStream().forEach(tl -> tl.searchByAccountFid(taskId, foundFids, tableFileMap, tableMetaDataMap));
} catch (IOException e) {
throw new InternalException("Failed to read cursor " + e);
}
} else if (searchParameter == SearchParameter.ACCT_NUM) {
if (IndicSearchCommon.isCastableField(getTableName(), "ACCT_NUM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ACCT_NUM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByAcctNumAndMemberCodeFromAccount(searchValue, memberCode)) {
Set<Integer> foundFids = new HashSet<>();
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap));
accountSubTableLoaders.parallelStream().forEach(tl -> tl.searchByAccountFid(taskId, foundFids, tableFileMap, tableMetaDataMap));
} catch (IOException e) {
throw new InternalException("Failed to read cursor " + e);
}
} else if (searchParameter == SearchParameter.SERIAL_NUM_ACCT) {
if (IndicSearchCommon.isCastableField(getTableName(), "SERIAL_NUM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("SERIAL_NUM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllBySerialNumAndMemberCodeFromAccount(searchValue, memberCode)) {
Set<Integer> foundFids = new HashSet<>();
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap));
accountSubTableLoaders.parallelStream().forEach(tl -> tl.searchByAccountFid(taskId, foundFids, tableFileMap, tableMetaDataMap));

9
src/main/java/ru/nbch/indicsearchservice/loader/IdTableLoader.java
View file

@ -16,6 +16,7 @@ import ru.nbch.indicsearchservice.exception.InternalException;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.util.IndicSearchCommon;
import ru.nbch.indicsearchservice.util.MapperHolder;
@Component
@ -66,6 +67,10 @@ public class IdTableLoader implements FirstLevelTableLoader {
if (searchParameter == SearchParameter.OGRN || searchParameter == SearchParameter.PASSPORT) {
Set<Integer> foundFids = new HashSet<>();
if (IndicSearchCommon.isCastableField(getTableName(), "ID_NORM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ID_NORM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = searchParameter == SearchParameter.OGRN ?
mapperHolder.getIndicMapper().findFidsByOgrnFromId(searchValue) :
mapperHolder.getIndicMapper().findFidsByPassportFromId(searchValue)) {
@ -99,6 +104,10 @@ public class IdTableLoader implements FirstLevelTableLoader {
if (searchParameter == SearchParameter.OGRN || searchParameter == SearchParameter.PASSPORT) {
Set<Integer> foundFids = new HashSet<>();
if (IndicSearchCommon.isCastableField(getTableName(), "ID_NORM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ID_NORM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = searchParameter == SearchParameter.OGRN ?
mapperHolder.getIndicMapper().findFidsByOgrnFromIdForTutdf(searchValue) :
mapperHolder.getIndicMapper().findFidsByPassportFromIdForTutdf(searchValue)) {

19
src/main/java/ru/nbch/indicsearchservice/loader/InformationTableLoader.java
View file

@ -13,6 +13,7 @@ import ru.nbch.indicsearchservice.exception.InternalException;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.util.IndicSearchCommon;
import ru.nbch.indicsearchservice.util.IndicSearchUtils;
import ru.nbch.indicsearchservice.util.MapperHolder;
@ -56,8 +57,22 @@ public class InformationTableLoader implements FirstLevelTableLoader, TableLoade
Map<String, Path> tableFileMap,
Map<String, TableMetaData> tableMetaDataMap) {
if (dataType == DataType.TUTDF) {
if (searchParameter == SearchParameter.OLD_APP_NUM || searchParameter == SearchParameter.SERIAL_NUM_APP) {
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromInformation(memberCode)) {
if (searchParameter == SearchParameter.OLD_APP_NUM) {
if (IndicSearchCommon.isCastableField(getTableName(), "APP_NUM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("APP_NUM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByAppNumAndMemberCodeFromInformation(searchValue, memberCode)) {
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap));
} catch (IOException e) {
throw new InternalException("Failed to read cursor " + e);
}
} else if (searchParameter == SearchParameter.SERIAL_NUM_APP) {
if (IndicSearchCommon.isCastableField(getTableName(), "SERIAL_NUM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("SERIAL_NUM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllBySerialNumAndMemberCodeFromInformation(searchValue, memberCode)) {
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap));
} catch (IOException e) {
throw new InternalException("Failed to read cursor " + e);

19
src/main/java/ru/nbch/indicsearchservice/loader/RuApplicationTableLoader.java
View file

@ -13,6 +13,7 @@ import ru.nbch.indicsearchservice.exception.InternalException;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.util.IndicSearchCommon;
import ru.nbch.indicsearchservice.util.IndicSearchUtils;
import ru.nbch.indicsearchservice.util.MapperHolder;
@ -56,8 +57,22 @@ public class RuApplicationTableLoader implements FirstLevelTableLoader, TableLoa
Map<String, Path> tableFileMap,
Map<String, TableMetaData> tableMetaDataMap) {
if (dataType == DataType.GUTDF || dataType == DataType.RUTDF) {
if (searchParameter == SearchParameter.APPLICATION_NUMBER || searchParameter == SearchParameter.OLD_APP_NUM) {
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromRuApplication(memberCode)) {
if (searchParameter == SearchParameter.APPLICATION_NUMBER) {
if (IndicSearchCommon.isCastableField(getTableName(), "APPLICATION_NUMBER")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("APPLICATION_NUMBER").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByApplicationNumberAndMemberCodeFromRuApplication(searchValue, memberCode)) {
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap));
} catch (IOException e) {
throw new InternalException("Failed to read cursor " + e);
}
} else if (searchParameter == SearchParameter.OLD_APP_NUM) {
if (IndicSearchCommon.isCastableField(getTableName(), "OLD_APP_NUM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("OLD_APP_NUM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByOldAppNumAndMemberCodeFromRuApplication(searchValue, memberCode)) {
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap));
} catch (IOException e) {
throw new InternalException("Failed to read cursor " + e);

21
src/main/java/ru/nbch/indicsearchservice/loader/RuUidTableLoader.java
View file

@ -16,6 +16,7 @@ import ru.nbch.indicsearchservice.loader.subtable.RuUidSubTableLoader;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.util.IndicSearchCommon;
import ru.nbch.indicsearchservice.util.IndicSearchUtils;
import ru.nbch.indicsearchservice.util.MapperHolder;
@ -67,8 +68,24 @@ public class RuUidTableLoader implements FirstLevelTableLoader, TableLoaderForId
Map<String, Path> tableFileMap,
Map<String, TableMetaData> tableMetaDataMap) {
if (dataType == DataType.GUTDF || dataType == DataType.RUTDF) {
if (searchParameter == SearchParameter.UUID || searchParameter == SearchParameter.ACCT_NUM) {
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromRuUid(memberCode)) {
if (searchParameter == SearchParameter.UUID) {
if (IndicSearchCommon.isCastableField(getTableName(), "UUID")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("UUID").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByUuidAndMemberCodeFromRuUid(searchValue, memberCode)) {
Set<Integer> foundFids = new HashSet<>();
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap));
ruUidSubTableLoaders.parallelStream().forEach(tl -> tl.searchByRuUidFid(taskId, foundFids, tableFileMap, tableMetaDataMap));
} catch (IOException e) {
throw new InternalException("Failed to read cursor " + e);
}
} else if (searchParameter == SearchParameter.ACCT_NUM) {
if (IndicSearchCommon.isCastableField(getTableName(), "ACCT_NUM")) {
int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ACCT_NUM").getColumnSize();
searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue;
}
try (Cursor<LinkedHashMap<String, Object>> cursor = mapperHolder.getIndicMapper().findAllByAcctNumAndMemberCodeFromRuUid(searchValue, memberCode)) {
Set<Integer> foundFids = new HashSet<>();
tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap));
ruUidSubTableLoaders.parallelStream().forEach(tl -> tl.searchByRuUidFid(taskId, foundFids, tableFileMap, tableMetaDataMap));

51
src/main/java/ru/nbch/indicsearchservice/model/TableMetaData.java
View file

@ -1,5 +1,7 @@
package ru.nbch.indicsearchservice.model;
import jakarta.annotation.Nullable;
import java.sql.JDBCType;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
@ -43,9 +45,54 @@ public class TableMetaData {
);
}
public record FieldMetaData(String fieldName, String fieldLocalization) {
public static class FieldMetaData {
private final String fieldName;
@Nullable
private final String fieldLocalization;
@Nullable
private JDBCType jdbcType;
@Nullable
private Integer columnSize;
public FieldMetaData(String fieldName, @Nullable String fieldLocalization) {
this.fieldName = fieldName;
this.fieldLocalization = fieldLocalization;
this.jdbcType = null;
this.columnSize = null;
}
public FieldMetaData(String fieldName) {
this(fieldName, null);
this.fieldName = fieldName;
this.fieldLocalization = null;
this.jdbcType = null;
this.columnSize = null;
}
public String getFieldName() {
return fieldName;
}
@Nullable
public String getFieldLocalization() {
return fieldLocalization;
}
@Nullable
public JDBCType getJdbcType() {
return jdbcType;
}
@Nullable
public Integer getColumnSize() {
return columnSize;
}
public void setJdbcType(@Nullable JDBCType jdbcType) {
this.jdbcType = jdbcType;
}
public void setColumnSize(@Nullable Integer columnSize) {
this.columnSize = columnSize;
}
}
}

15
src/main/java/ru/nbch/indicsearchservice/repository/indic/IndicMapper.java
View file

@ -21,8 +21,11 @@ public interface IndicMapper extends MyBatisRepository {
Cursor<LinkedHashMap<String, Object>> findAllByFidsFromRuCapability(Set<Integer> fids);
Cursor<LinkedHashMap<String, Object>> findAllByFidsFromRuDebtBurdenInfo(Set<Integer> fids);
Cursor<LinkedHashMap<String, Object>> findAllByMemberCodeFromRuUid(String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByMemberCodeFromRuApplication(String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByAcctNumAndMemberCodeFromRuUid(String acctNum, String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByUuidAndMemberCodeFromRuUid(String uuid, String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByApplicationNumberAndMemberCodeFromRuApplication(String applicationNumber, String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByOldAppNumAndMemberCodeFromRuApplication(String oldAppNum, String memberCode);
Cursor<LinkedHashMap<String, Object>> findFidsByOgrnFromId(String idNorm);
Cursor<LinkedHashMap<String, Object>> findFidsByPassportFromId(String idNorm);
@ -68,8 +71,12 @@ public interface IndicMapper extends MyBatisRepository {
Cursor<LinkedHashMap<String, Object>> findAllByFidsFromConsumerBankruptcy(Set<Integer> fids);
Cursor<LinkedHashMap<String, Object>> findAllByFidsFromLegal(Set<Integer> fids);
Cursor<LinkedHashMap<String, Object>> findAllByMemberCodeFromAccount(String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByMemberCodeFromInformation(String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByAcctNumAndMemberCodeFromAccount(String acctNum, String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllBySerialNumAndMemberCodeFromAccount(String serialNum, String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByUuidAndMemberCodeFromAccount(String uuid, String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllBySerialNumAndMemberCodeFromInformation(String serialNum, String memberCode);
Cursor<LinkedHashMap<String, Object>> findAllByAppNumAndMemberCodeFromInformation(String appNum, String memberCode);
Cursor<LinkedHashMap<String, Object>> findFidsByOgrnFromIdForTutdf(String idNorm);
Cursor<LinkedHashMap<String, Object>> findFidsByPassportFromIdForTutdf(String idNorm);

188
src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java
View file

@ -3,18 +3,11 @@ package ru.nbch.indicsearchservice.service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.mutable.MutableLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -29,15 +22,8 @@ import ru.nbch.indicsearchservice.exception.InternalException;
import ru.nbch.indicsearchservice.exporter.ExcelExporter;
import ru.nbch.indicsearchservice.exporter.JsonExporter;
import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader;
import ru.nbch.indicsearchservice.model.LocalizationObject;
import ru.nbch.indicsearchservice.model.Settings;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.repository.bank.LocalizationObjectMapper;
import ru.nbch.indicsearchservice.repository.bank.SettingsMapper;
import ru.nbch.indicsearchservice.util.IndicSearchUtils;
import ru.nbch.indicsearchservice.util.MapperHolder;
import ru.nbch.indicsearchservice.util.TableMetaDataUtil;
@Service
public class IndicSearchService {
@ -50,29 +36,20 @@ public class IndicSearchService {
private final ExcelExporter excelExporter;
private final JsonExporter jsonExporter;
private final SettingsService settingsService;
private final LocalizationObjectMapper localizationObjectMapper;
private final SettingsMapper settingsMapper;
private final MapperHolder mapperHolder;
private final IndicSearchUtils indicSearchUtils;
private final TableMetaDataUtil tableMetaDataUtil;
public IndicSearchService(List<FirstLevelTableLoader> tableLoaders,
List<FileAggregator> fileAggregators,
ExcelExporter excelExporter,
JsonExporter jsonExporter,
SettingsService settingsService,
LocalizationObjectMapper localizationObjectMapper,
SettingsMapper settingsMapper,
MapperHolder mapperHolder,
IndicSearchUtils indicSearchUtils) {
TableMetaDataUtil tableMetaDataUtil) {
this.tableLoaders = tableLoaders;
this.fileAggregators = fileAggregators;
this.excelExporter = excelExporter;
this.jsonExporter = jsonExporter;
this.settingsService = settingsService;
this.localizationObjectMapper = localizationObjectMapper;
this.settingsMapper = settingsMapper;
this.mapperHolder = mapperHolder;
this.indicSearchUtils = indicSearchUtils;
this.tableMetaDataUtil = tableMetaDataUtil;
}
public AddSearchJobResponseDto addSearchJob(AddSearchJobRequestDto request) {
@ -84,7 +61,7 @@ public class IndicSearchService {
log.info("Prep step started. Getting metadata from database");
sw.start();
Map<String, TableMetaData> tableMetaDataMap = prepareTableMetaData(request.getDataType(), request.getSearchParameter());
Map<String, TableMetaData> tableMetaDataMap = tableMetaDataUtil.prepareTableMetaData(request.getDataType(), request.getSearchParameter());
long rowLimit = Long.parseLong(settingsService.getSettingsValueByName("LimitDB"));
sw.stop();
@ -123,6 +100,7 @@ public class IndicSearchService {
request.getMemberCode().substring(0, 6),
tableFileMap,
rowCounter))
.filter(Objects::nonNull)
.findFirst()
.orElse(null);
@ -145,7 +123,7 @@ public class IndicSearchService {
if (rowCounter.longValue() < rowLimit && !IS_EXPORT_AS_EXCEL) {
jsonExporter.writeJsonFile(aggregatedFile, taskId, request.getSearchParameter(), tableMetaDataMap);
} else {
fillTableMetaDataWithLocalization(tableMetaDataMap);
tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap);
excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue(), tableMetaDataMap);
}
@ -170,154 +148,4 @@ public class IndicSearchService {
public ResponseEntity<?> export(Long id, String exportType) {
return null;
}
private Map<String, TableMetaData> prepareTableMetaData(DataType dataType, SearchParameter searchParameter) {
Map<String, TableMetaData> tableMetaDataMap = new ConcurrentHashMap<>();
// Получаем список таблиц
Set<String> tableNames = tableLoaders.parallelStream()
.map(tl -> tl.getTableNamesIfSuitable(dataType, searchParameter))
.flatMap(Set::stream)
.collect(Collectors.toSet());
if (tableNames.isEmpty()) {
return tableMetaDataMap;
}
tableNames.forEach(name -> tableMetaDataMap.put(name, new TableMetaData(name)));
// Составляем массив ключей из SETTINGS
Map<String, String> keyToTable = new HashMap<>();
List<String> columnKeys = new ArrayList<>();
for (String table : tableNames) {
String key = "db.table." + table.toUpperCase() + ".fields";
columnKeys.add(key);
keyToTable.put(key, table);
}
// Получаем данные из SETTINGS одним запросом
Map<String, Optional<String>> settingsValues = settingsService.getSettingsValuesByNames(columnKeys);
Set<String> missingTables = new HashSet<>();
List<Settings> batchInsertSettings = new ArrayList<>();
for (String key : columnKeys) {
String table = keyToTable.get(key);
Optional<String> valueOpt = settingsValues.get(key);
if (valueOpt.isPresent() && StringUtils.isNotBlank(valueOpt.get())) {
String[] fields = valueOpt.get().split(";");
for (String field : fields) {
tableMetaDataMap.get(table).addField(field.trim());
}
} else {
missingTables.add(table);
}
}
// Если найдены пропущенные значения, то добавляем их в БД
if (!missingTables.isEmpty()) {
List<LinkedHashMap<String, String>> rows =
mapperHolder.getIndicMapper().getTableColumnsByTableNames(new ArrayList<>(missingTables));
Map<String, List<String>> tableToColumns = new HashMap<>();
for (Map<String, String> row : rows) {
tableToColumns.computeIfAbsent(row.get("TBNAME"), t -> new ArrayList<>()).add(row.get("NAME"));
}
for (String table : missingTables) {
List<String> columns = tableToColumns.getOrDefault(table, List.of());
String key = "db.table." + table.toUpperCase() + ".fields";
String value = String.join(";", columns);
Settings newSettings = new Settings();
newSettings.setName(key);
newSettings.setValue(value);
batchInsertSettings.add(newSettings);
for (String column : columns) {
tableMetaDataMap.get(table).addField(column.trim());
}
}
if (!batchInsertSettings.isEmpty()) {
settingsMapper.insertBatch(batchInsertSettings);
}
}
return tableMetaDataMap;
}
private void fillTableMetaDataWithLocalization(Map<String, TableMetaData> tableMetaDataMap) {
if (tableMetaDataMap == null || tableMetaDataMap.isEmpty()) {
return;
}
// Получаем все ключи для LOCALIZATION_OBJECT
Set<String> localizationKeys = new HashSet<>();
for (TableMetaData metaData : tableMetaDataMap.values()) {
String tableUpper = metaData.getTableName().toUpperCase();
localizationKeys.add("db.viewer.table." + tableUpper);
for (String field : metaData.getFieldMetaData().keySet()) {
String fieldUpper = field.toUpperCase();
localizationKeys.add("db.viewer." + tableUpper + ".field." + fieldUpper);
localizationKeys.add("db.viewer.field." + fieldUpper);
}
}
// Получаем данные из LOCALIZATION_OBJECT одним запросом
Map<String, Optional<String>> localizationValues = indicSearchUtils.getOptionalLocalizationStrings(new ArrayList<>(localizationKeys));
List<LocalizationObject> toInsertLocalizations = new ArrayList<>();
// Вставляем локализацию в tableMetaDataMap
for (TableMetaData metaData : tableMetaDataMap.values()) {
String tableUpper = metaData.getTableName().toUpperCase();
String tableKey = "db.viewer.table." + tableUpper;
// Локализация для таблиц
Optional<String> tableLoc = localizationValues.get(tableKey);
if (tableLoc.isPresent() && StringUtils.isNotBlank(tableLoc.get())) {
metaData.setTableNameLocalization(tableLoc.get());
} else {
String fallbackText = "Таблица " + tableUpper;
metaData.setTableNameLocalization(fallbackText);
LocalizationObject newLocalizationObject = new LocalizationObject();
newLocalizationObject.setCode(tableKey);
newLocalizationObject.setText(fallbackText);
toInsertLocalizations.add(newLocalizationObject);
}
// Локализация для полей
for (String field : metaData.getFieldMetaData().keySet()) {
String fieldUpper = field.toUpperCase();
String primaryKey = "db.viewer." + tableUpper + ".field." + fieldUpper;
String fallbackKey = "db.viewer.field." + fieldUpper;
Optional<String> primaryLoc = localizationValues.get(primaryKey);
if (primaryLoc.isPresent() && StringUtils.isNotBlank(primaryLoc.get())) {
metaData.addLocalization(field, primaryLoc.get());
} else {
Optional<String> fallbackLoc = localizationValues.get(fallbackKey);
if (fallbackLoc.isPresent() && StringUtils.isNotBlank(fallbackLoc.get())) {
metaData.addLocalization(field, fallbackLoc.get());
} else {
String fallbackText = "Поле " + fieldUpper;
metaData.addLocalization(field, fallbackText);
LocalizationObject newLocalizationObject = new LocalizationObject();
newLocalizationObject.setCode(primaryKey);
newLocalizationObject.setText(fallbackText);
toInsertLocalizations.add(newLocalizationObject);
}
}
}
}
// Если найдены пропущенные значения, то добавляем их в БД
if (!toInsertLocalizations.isEmpty()) {
localizationObjectMapper.insertBatch(toInsertLocalizations);
}
}
}

25
src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java
View file

@ -2,9 +2,34 @@ package ru.nbch.indicsearchservice.util;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public class IndicSearchCommon {
private static final Map<String, List<String>> CASTABLE_FIELDS = Map.ofEntries(
Map.entry("RU_UID", List.of("ACCT_NUM")),
Map.entry("RU_APPLICATION", List.of("OLD_APP_NUM")),
Map.entry("ID", List.of("ID_NORM")),
Map.entry("ACCOUNT", List.of("ACCT_NUM")),
Map.entry("INFORMATION", List.of("APP_NUM"))
);
public static Map<String, List<String>> getCastableFields() {
return CASTABLE_FIELDS;
}
public static boolean isCastableField(String tableName, String fieldName) {
if (tableName == null || tableName.isEmpty() || fieldName == null || fieldName.isEmpty()) {
return false;
}
if (CASTABLE_FIELDS.containsKey(tableName.toUpperCase())) {
return CASTABLE_FIELDS.get(tableName).contains(fieldName.toUpperCase());
}
return false;
}
public static Optional<String> getRowValue(String string, String key) {
return getRowValue(Arrays.asList(string.split("%%%", -1)), key);
}

233
src/main/java/ru/nbch/indicsearchservice/util/TableMetaDataUtil.java Normal file
View file

@ -0,0 +1,233 @@
package ru.nbch.indicsearchservice.util;
import java.sql.Connection;
import java.sql.JDBCType;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.datasource.DataSourceUtils;
import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader;
import ru.nbch.indicsearchservice.model.LocalizationObject;
import ru.nbch.indicsearchservice.model.Settings;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.repository.bank.LocalizationObjectMapper;
import ru.nbch.indicsearchservice.repository.bank.SettingsMapper;
import ru.nbch.indicsearchservice.service.SettingsService;
@Component
public class TableMetaDataUtil {
private final List<FirstLevelTableLoader> tableLoaders;
private final SettingsService settingsService;
private final MapperHolder mapperHolder;
private final SettingsMapper settingsMapper;
private final LocalizationObjectMapper localizationObjectMapper;
private final IndicSearchUtils indicSearchUtils;
private final DataSource indicDataSource;
public TableMetaDataUtil(List<FirstLevelTableLoader> tableLoaders,
SettingsService settingsService,
MapperHolder mapperHolder,
SettingsMapper settingsMapper,
LocalizationObjectMapper localizationObjectMapper,
IndicSearchUtils indicSearchUtils,
@Qualifier("indicDataSource") DataSource indicDataSource) {
this.tableLoaders = tableLoaders;
this.settingsService = settingsService;
this.mapperHolder = mapperHolder;
this.settingsMapper = settingsMapper;
this.localizationObjectMapper = localizationObjectMapper;
this.indicSearchUtils = indicSearchUtils;
this.indicDataSource = indicDataSource;
}
public Map<String, TableMetaData> prepareTableMetaData(DataType dataType, SearchParameter searchParameter) {
Map<String, TableMetaData> tableMetaDataMap = new ConcurrentHashMap<>();
// Получаем список таблиц
Set<String> tableNames = tableLoaders.parallelStream()
.map(tl -> tl.getTableNamesIfSuitable(dataType, searchParameter))
.flatMap(Set::stream)
.collect(Collectors.toSet());
if (tableNames.isEmpty()) {
return tableMetaDataMap;
}
tableNames.forEach(name -> tableMetaDataMap.put(name, new TableMetaData(name)));
// Составляем массив ключей из SETTINGS
Map<String, String> keyToTable = new HashMap<>();
List<String> columnKeys = new ArrayList<>();
for (String table : tableNames) {
String key = "db.table." + table.toUpperCase() + ".fields";
columnKeys.add(key);
keyToTable.put(key, table);
}
// Получаем данные из SETTINGS одним запросом
Map<String, Optional<String>> settingsValues = settingsService.getSettingsValuesByNames(columnKeys);
Set<String> missingTables = new HashSet<>();
List<Settings> batchInsertSettings = new ArrayList<>();
for (String key : columnKeys) {
String table = keyToTable.get(key);
Optional<String> valueOpt = settingsValues.get(key);
if (valueOpt.isPresent() && StringUtils.isNotBlank(valueOpt.get())) {
String[] fields = valueOpt.get().split(";");
for (String field : fields) {
tableMetaDataMap.get(table).addField(field.trim());
}
} else {
missingTables.add(table);
}
}
// Если найдены пропущенные значения, то добавляем их в БД
if (!missingTables.isEmpty()) {
List<LinkedHashMap<String, String>> rows =
mapperHolder.getIndicMapper().getTableColumnsByTableNames(new ArrayList<>(missingTables));
Map<String, List<String>> tableToColumns = new HashMap<>();
for (Map<String, String> row : rows) {
tableToColumns.computeIfAbsent(row.get("TBNAME"), t -> new ArrayList<>()).add(row.get("NAME"));
}
for (String table : missingTables) {
List<String> columns = tableToColumns.getOrDefault(table, List.of());
String key = "db.table." + table.toUpperCase() + ".fields";
String value = String.join(";", columns);
Settings newSettings = new Settings();
newSettings.setName(key);
newSettings.setValue(value);
batchInsertSettings.add(newSettings);
for (String column : columns) {
tableMetaDataMap.get(table).addField(column.trim());
}
}
if (!batchInsertSettings.isEmpty()) {
settingsMapper.insertBatch(batchInsertSettings);
}
}
// Получаем размеры некоторых полей
for (Map.Entry<String, List<String>> entry : IndicSearchCommon.getCastableFields().entrySet()) {
String tableName = entry.getKey();
List<String> tableFields = entry.getValue();
if (!tableMetaDataMap.containsKey(tableName)) {
continue;
}
try (Connection connection = DataSourceUtils.getConnection(indicDataSource);
ResultSet resultSet = connection.getMetaData().getColumns(null, null, tableName, null)) {
while (resultSet.next()) {
String columnName = resultSet.getString("COLUMN_NAME");
if (tableFields.contains(columnName) && tableMetaDataMap.get(tableName).getFieldMetaData().containsKey(columnName)) {
TableMetaData.FieldMetaData foundFieldMetaData = tableMetaDataMap.get(tableName).getFieldMetaData().get(columnName);
foundFieldMetaData.setJdbcType(JDBCType.valueOf(resultSet.getInt("DATA_TYPE")));
foundFieldMetaData.setColumnSize(resultSet.getInt("COLUMN_SIZE"));
}
}
DataSourceUtils.releaseConnection(connection, indicDataSource);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
return tableMetaDataMap;
}
public void fillTableMetaDataWithLocalization(Map<String, TableMetaData> tableMetaDataMap) {
if (tableMetaDataMap == null || tableMetaDataMap.isEmpty()) {
return;
}
// Получаем все ключи для LOCALIZATION_OBJECT
Set<String> localizationKeys = new HashSet<>();
for (TableMetaData metaData : tableMetaDataMap.values()) {
String tableUpper = metaData.getTableName().toUpperCase();
localizationKeys.add("db.viewer.table." + tableUpper);
for (String field : metaData.getFieldMetaData().keySet()) {
String fieldUpper = field.toUpperCase();
localizationKeys.add("db.viewer." + tableUpper + ".field." + fieldUpper);
localizationKeys.add("db.viewer.field." + fieldUpper);
}
}
// Получаем данные из LOCALIZATION_OBJECT одним запросом
Map<String, Optional<String>> localizationValues = indicSearchUtils.getOptionalLocalizationStrings(new ArrayList<>(localizationKeys));
List<LocalizationObject> toInsertLocalizations = new ArrayList<>();
// Вставляем локализацию в tableMetaDataMap
for (TableMetaData metaData : tableMetaDataMap.values()) {
String tableUpper = metaData.getTableName().toUpperCase();
String tableKey = "db.viewer.table." + tableUpper;
// Локализация для таблиц
Optional<String> tableLoc = localizationValues.get(tableKey);
if (tableLoc.isPresent() && StringUtils.isNotBlank(tableLoc.get())) {
metaData.setTableNameLocalization(tableLoc.get());
} else {
String fallbackText = "Таблица " + tableUpper;
metaData.setTableNameLocalization(fallbackText);
LocalizationObject newLocalizationObject = new LocalizationObject();
newLocalizationObject.setCode(tableKey);
newLocalizationObject.setText(fallbackText);
toInsertLocalizations.add(newLocalizationObject);
}
// Локализация для полей
for (String field : metaData.getFieldMetaData().keySet()) {
String fieldUpper = field.toUpperCase();
String primaryKey = "db.viewer." + tableUpper + ".field." + fieldUpper;
String fallbackKey = "db.viewer.field." + fieldUpper;
Optional<String> primaryLoc = localizationValues.get(primaryKey);
if (primaryLoc.isPresent() && StringUtils.isNotBlank(primaryLoc.get())) {
metaData.addLocalization(field, primaryLoc.get());
} else {
Optional<String> fallbackLoc = localizationValues.get(fallbackKey);
if (fallbackLoc.isPresent() && StringUtils.isNotBlank(fallbackLoc.get())) {
metaData.addLocalization(field, fallbackLoc.get());
} else {
String fallbackText = "Поле " + fieldUpper;
metaData.addLocalization(field, fallbackText);
LocalizationObject newLocalizationObject = new LocalizationObject();
newLocalizationObject.setCode(primaryKey);
newLocalizationObject.setText(fallbackText);
toInsertLocalizations.add(newLocalizationObject);
}
}
}
}
// Если найдены пропущенные значения, то добавляем их в БД
if (!toInsertLocalizations.isEmpty()) {
localizationObjectMapper.insertBatch(toInsertLocalizations);
}
}
}

83
src/main/resources/mappers/indic/indic_mapper.xml
View file

@ -102,30 +102,52 @@
WITH UR
</select>
<select id="findAllByMemberCodeFromRuUid" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
<select id="findAllByAcctNumAndMemberCodeFromRuUid" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.RU_UID
WHERE (MEMBER_CODE LIKE #{memberCode} || '%'
WHERE ACCT_NUM = #{acctNum}
AND (MEMBER_CODE LIKE #{memberCode} || '%'
OR REAL_MEMBERCODE LIKE #{memberCode} || '%')
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findAllByMemberCodeFromRuApplication" resultType="java.util.LinkedHashMap" fetchSize="1000"
timeout="3600">
<select id="findAllByUuidAndMemberCodeFromRuUid" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.RU_UID
WHERE UUID = #{uuid}
AND (MEMBER_CODE LIKE #{memberCode} || '%'
OR REAL_MEMBERCODE LIKE #{memberCode} || '%')
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findAllByApplicationNumberAndMemberCodeFromRuApplication" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.RU_APPLICATION
WHERE (MEMBER_CODE LIKE #{memberCode} || '%'
WHERE APPLICATION_NUMBER = #{applicationNumber}
AND (MEMBER_CODE LIKE #{memberCode} || '%'
OR REAL_MEMBERCODE LIKE #{memberCode} || '%')
AND (CANCELLATION_FLAG != 'Y' OR CANCELLATION_FLAG IS NULL)
ORDER BY LAST_UPDATED_DT DESC
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findAllByOldAppNumAndMemberCodeFromRuApplication" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.RU_APPLICATION
WHERE OLD_APP_NUM = #{oldAppNum}
AND (MEMBER_CODE LIKE #{memberCode} || '%'
OR REAL_MEMBERCODE LIKE #{memberCode} || '%')
AND (CANCELLATION_FLAG != 'Y' OR CANCELLATION_FLAG IS NULL)
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findFidsByOgrnFromId" resultType="java.util.LinkedHashMap" timeout="3600">
SELECT FID
FROM INDIC.ID
WHERE CAST(ID_NORM AS VARCHAR(2000)) = #{idNorm}
WHERE ID_NORM = #{idNorm}
AND (ID_TYPE LIKE '34' OR ID_TYPE LIKE '35' OR ID_TYPE2 LIKE 'regnumUL')
ORDER BY LAST_UPDATED_DT DESC
WITH UR
@ -134,7 +156,7 @@
<select id="findFidsByPassportFromId" resultType="java.util.LinkedHashMap" timeout="3600">
SELECT FID
FROM INDIC.ID
WHERE CAST(ID_NORM AS VARCHAR(2000)) = #{idNorm}
WHERE ID_NORM = #{idNorm}
AND (ID_TYPE LIKE '21' OR ID_TYPE2 LIKE '21' OR ID_TYPE LIKE '10' OR ID_TYPE2 LIKE '31')
ORDER BY LAST_UPDATED_DT DESC
WITH UR
@ -578,28 +600,57 @@
WITH UR
</select>
<select id="findAllByMemberCodeFromAccount" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
<select id="findAllByAcctNumAndMemberCodeFromAccount" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.ACCOUNT
WHERE MEMBER_CODE LIKE #{memberCode} || '%'
WHERE ACCT_NUM = #{acctNum}
AND MEMBER_CODE LIKE #{memberCode} || '%'
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findAllByMemberCodeFromInformation" resultType="java.util.LinkedHashMap" fetchSize="1000"
timeout="3600">
<select id="findAllBySerialNumAndMemberCodeFromAccount" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.ACCOUNT
WHERE SERIAL_NUM = #{serialNum}
AND MEMBER_CODE LIKE #{memberCode} || '%'
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findAllByUuidAndMemberCodeFromAccount" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.ACCOUNT
WHERE UUID = #{uuid}
AND MEMBER_CODE LIKE #{memberCode} || '%'
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findAllBySerialNumAndMemberCodeFromInformation" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.INFORMATION
WHERE MEMBER_CODE LIKE #{memberCode} || '%'
WHERE SERIAL_NUM = #{serialNum}
AND MEMBER_CODE LIKE #{memberCode} || '%'
AND (CANCELLATION_FLAG != 'Y' OR CANCELLATION_FLAG IS NULL)
ORDER BY LAST_UPDATED_DT DESC
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findAllByAppNumAndMemberCodeFromInformation" resultType="java.util.LinkedHashMap" fetchSize="1000" timeout="3600">
SELECT *
FROM INDIC.INFORMATION
WHERE APP_NUM = #{appNum}
AND MEMBER_CODE LIKE #{memberCode} || '%'
AND (CANCELLATION_FLAG != 'Y' OR CANCELLATION_FLAG IS NULL)
ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC
WITH UR
</select>
<select id="findFidsByOgrnFromIdForTutdf" resultType="java.util.LinkedHashMap" timeout="3600">
SELECT FID
FROM INDIC.ID
WHERE CAST(ID_NORM AS VARCHAR(2000)) = #{idNorm}
WHERE ID_NORM = #{idNorm}
AND ID_TYPE in ('34')
ORDER BY LAST_UPDATED_DT DESC
WITH UR
@ -608,7 +659,7 @@
<select id="findFidsByPassportFromIdForTutdf" resultType="java.util.LinkedHashMap" timeout="3600">
SELECT FID
FROM INDIC.ID
WHERE CAST(ID_NORM AS VARCHAR(2000)) = #{idNorm}
WHERE ID_NORM = #{idNorm}
AND ID_TYPE in ('10', '21')
ORDER BY LAST_UPDATED_DT DESC
WITH UR

20
src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java
View file

@ -9,6 +9,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
@ -19,13 +20,18 @@ import ru.nbch.indicsearchservice.model.enums.SearchParameter;
@SpringBootTest
class JsonExporterTest {
@Autowired
ApplicationSettings applicationSettings;
ApplicationSettings settings;
@Autowired
JsonExporter jsonExporter;
@BeforeEach
void setUp() {
settings.setStorageDirectory(Path.of("src/test/resources/storage").toFile());
}
@Test
void writeJsonFile_shouldCreateJsonFilePASSPORT() throws IOException {
Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv");
Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("test-json-aggregated.csv");
UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907");
TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID");
@ -41,16 +47,18 @@ class JsonExporterTest {
jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.PASSPORT, tableMetaDataMap);
Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json");
Path resultFile = settings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json");
List<String> resultFileStrings = Files.readAllLines(resultFile);
Files.deleteIfExists(resultFile);
assertThat(resultFileStrings.getFirst())
.isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]},{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}");
}
@Test
void writeJsonFile_shouldCreateJsonFileUUID() throws IOException {
Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv");
Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("test-json-aggregated.csv");
UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907");
TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID");
@ -66,9 +74,11 @@ class JsonExporterTest {
jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.UUID, tableMetaDataMap);
Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json");
Path resultFile = settings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json");
List<String> resultFileStrings = Files.readAllLines(resultFile);
Files.deleteIfExists(resultFile);
assertThat(resultFileStrings.getFirst())
.isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]}],[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}");
}