diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java index b79e20d..44e9f3a 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java @@ -63,8 +63,6 @@ public class FileAggregatorCommon { void aggregateRuUidFiles(BufferedWriter aggregatedFileWriter, Map tableFileMap, String memberCode, - String acctNum, - String uuid, MutableLong rowCounter) { try (BufferedReader ruUidFileReader = Files.newBufferedReader(tableFileMap.get("RU_UID"))) { Map childTablesReadersMap = new LinkedHashMap<>(); @@ -88,21 +86,6 @@ public class FileAggregatorCommon { continue; } - if (acctNum != null) { - Optional foundAcctNum = IndicSearchCommon.getRowValue(lineList, "ACCT_NUM"); - - if (foundAcctNum.isPresent() && !foundAcctNum.get().equals(acctNum)) { - continue; - } - } - if (uuid != null) { - Optional foundUuid = IndicSearchCommon.getRowValue(lineList, "UUID"); - - if (foundUuid.isPresent() && !foundUuid.get().equals(uuid)) { - continue; - } - } - aggregatedFileWriter.write(ruUidLine); aggregatedFileWriter.newLine(); rowCounter.increment(); @@ -281,9 +264,6 @@ public class FileAggregatorCommon { void aggregateAccountFiles(BufferedWriter aggregatedFileWriter, Map tableFileMap, String memberCode, - String acctNum, - String uuid, - String serialNum, MutableLong rowCounter) { try (BufferedReader accountFileReader = Files.newBufferedReader(tableFileMap.get("ACCOUNT"))) { Map childTablesReadersMap = new LinkedHashMap<>(); @@ -307,32 +287,11 @@ public class FileAggregatorCommon { continue; } - if (acctNum != null) { - Optional foundAcctNum = IndicSearchCommon.getRowValue(lineList, "ACCT_NUM"); - - if (foundAcctNum.isPresent() && !foundAcctNum.get().equals(acctNum)) { - continue; - } - } - if (uuid != null) { - Optional foundUuid = IndicSearchCommon.getRowValue(lineList, "UUID"); - - if (foundUuid.isPresent() && !foundUuid.get().equals(uuid)) { - continue; - } - } - - Optional foundAccountSerialNum = IndicSearchCommon.getRowValue(lineList, "SERIAL_NUM"); - if (serialNum != null) { - if (foundAccountSerialNum.isPresent() && !foundAccountSerialNum.get().equals(serialNum)) { - continue; - } - } - aggregatedFileWriter.write(accountLine); aggregatedFileWriter.newLine(); rowCounter.increment(); + Optional foundAccountSerialNum = IndicSearchCommon.getRowValue(lineList, "SERIAL_NUM"); if (foundAccountSerialNum.isEmpty()) { throw new InternalException("SERIAL_NUM was not found in line " + accountLine); } diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidPassportOgrnFileAggregator.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidPassportOgrnFileAggregator.java index 46c24c2..6dede61 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidPassportOgrnFileAggregator.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfFidPassportOgrnFileAggregator.java @@ -55,7 +55,7 @@ public class GutdfFidPassportOgrnFileAggregator implements FileAggregator { try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { for (String tableName : REQUIRED_TABLES) { if (tableName.equals("RU_UID")) { - fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, null, null, rowCounter); + fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter); } else if (tableName.equals("RU_BANKRUPTCY")) { fileAggregatorCommon.aggregateBlocksTableFile(tableName, aggregatedFileWriter, tableFileMap, memberCode, rowCounter); } else { diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuApplicationFileAggregator.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuApplicationFileAggregator.java index 11a9590..354f76f 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuApplicationFileAggregator.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuApplicationFileAggregator.java @@ -42,23 +42,6 @@ public class GutdfRuApplicationFileAggregator implements FileAggregator { continue; } - if (searchParameter == SearchParameter.APPLICATION_NUMBER && searchValue != null) { - searchValue = searchValue.length() > 38 ? searchValue.substring(0, 38) : searchValue; - - Optional foundApplicationNumber = IndicSearchCommon.getRowValue(line, "APPLICATION_NUMBER"); - - if (foundApplicationNumber.isPresent() && !foundApplicationNumber.get().equals(searchValue)) { - continue; - } - } - if (searchParameter == SearchParameter.OLD_APP_NUM && searchValue != null) { - Optional foundOldAppNum = IndicSearchCommon.getRowValue(line, "OLD_APP_NUM"); - - if (foundOldAppNum.isPresent() && !foundOldAppNum.get().equals(searchValue)) { - continue; - } - } - aggregatedFileWriter.write(line); aggregatedFileWriter.newLine(); rowCounter.increment(); diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuUidFileAggregator.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuUidFileAggregator.java index 6d77190..5b2302e 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuUidFileAggregator.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/GutdfRuUidFileAggregator.java @@ -32,11 +32,7 @@ public class GutdfRuUidFileAggregator implements FileAggregator { Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("AGGREGATED-" + taskId + ".csv"); try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { - if (searchParameter == SearchParameter.UUID) { - fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, null, searchValue, rowCounter); - } else { - fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, searchValue, null, rowCounter); - } + fileAggregatorCommon.aggregateRuUidFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter); aggregatedFileWriter.write("EOF"); } catch (IOException e) { diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfAccountFileAggregator.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfAccountFileAggregator.java index 1b4f8ad..a6b470f 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfAccountFileAggregator.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfAccountFileAggregator.java @@ -32,13 +32,7 @@ public class TutdfAccountFileAggregator implements FileAggregator { Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("AGGREGATED-" + taskId + ".csv"); try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { - if (searchParameter == SearchParameter.UUID) { - fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, null, searchValue, null, rowCounter); - } else if (searchParameter == SearchParameter.ACCT_NUM) { - fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, searchValue, null, null, rowCounter); - } else { - fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, null, null, searchValue, rowCounter); - } + fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter); aggregatedFileWriter.write("EOF"); } catch (IOException e) { diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfFidPassportOgrnFileAggregator.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfFidPassportOgrnFileAggregator.java index 62c2e78..6a7c5fa 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfFidPassportOgrnFileAggregator.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfFidPassportOgrnFileAggregator.java @@ -52,7 +52,7 @@ public class TutdfFidPassportOgrnFileAggregator implements FileAggregator { try (BufferedWriter aggregatedFileWriter = Files.newBufferedWriter(aggregatedFile, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { for (String tableName : REQUIRED_TABLES) { if (tableName.equals("ACCOUNT")) { - fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, null, null, null, rowCounter); + fileAggregatorCommon.aggregateAccountFiles(aggregatedFileWriter, tableFileMap, memberCode, rowCounter); } else { try (BufferedReader reader = Files.newBufferedReader(tableFileMap.get(tableName))) { String line; diff --git a/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfInformationFileAggregator.java b/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfInformationFileAggregator.java index 3fe564b..e212950 100644 --- a/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfInformationFileAggregator.java +++ b/src/main/java/ru/nbch/indicsearchservice/aggregator/TutdfInformationFileAggregator.java @@ -42,21 +42,6 @@ public class TutdfInformationFileAggregator implements FileAggregator { continue; } - if (searchParameter == SearchParameter.OLD_APP_NUM && searchValue != null) { - Optional foundOldAppNum = IndicSearchCommon.getRowValue(line, "APP_NUM"); - - if (foundOldAppNum.isPresent() && !foundOldAppNum.get().equals(searchValue)) { - continue; - } - } - if (searchParameter == SearchParameter.SERIAL_NUM_APP && searchValue != null) { - Optional foundSerialNum = IndicSearchCommon.getRowValue(line, "SERIAL_NUM"); - - if (foundSerialNum.isPresent() && !foundSerialNum.get().equals(searchValue)) { - continue; - } - } - aggregatedFileWriter.write(line); aggregatedFileWriter.newLine(); rowCounter.increment(); diff --git a/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java index 78875ba..a701383 100644 --- a/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java @@ -137,7 +137,7 @@ public class ExcelExporter implements Exporter { String columnValue = rowSublist.get(i + 1); if (columnName != null && allowedFields.contains(columnName.toUpperCase())) { - rowValues.put(columnName, Pair.of(rowMetaData.getFieldMetaData().get(columnName).fieldLocalization(), columnValue)); + rowValues.put(columnName, Pair.of(rowMetaData.getFieldMetaData().get(columnName).getFieldLocalization(), columnValue)); } } diff --git a/src/main/java/ru/nbch/indicsearchservice/loader/AccountTableLoader.java b/src/main/java/ru/nbch/indicsearchservice/loader/AccountTableLoader.java index b620bf0..6b13950 100644 --- a/src/main/java/ru/nbch/indicsearchservice/loader/AccountTableLoader.java +++ b/src/main/java/ru/nbch/indicsearchservice/loader/AccountTableLoader.java @@ -15,6 +15,7 @@ import ru.nbch.indicsearchservice.loader.subtable.AccountSubTableLoader; import ru.nbch.indicsearchservice.model.TableMetaData; import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.util.IndicSearchCommon; import ru.nbch.indicsearchservice.util.IndicSearchUtils; import ru.nbch.indicsearchservice.util.MapperHolder; @@ -66,8 +67,36 @@ public class AccountTableLoader implements FirstLevelTableLoader, TableLoaderFor Map tableFileMap, Map tableMetaDataMap) { if (dataType == DataType.TUTDF) { - if (searchParameter == SearchParameter.UUID || searchParameter == SearchParameter.ACCT_NUM || searchParameter == SearchParameter.SERIAL_NUM_ACCT) { - try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromAccount(memberCode)) { + if (searchParameter == SearchParameter.UUID) { + if (IndicSearchCommon.isCastableField(getTableName(), "UUID")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("UUID").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByUuidAndMemberCodeFromAccount(searchValue, memberCode)) { + Set foundFids = new HashSet<>(); + tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap)); + accountSubTableLoaders.parallelStream().forEach(tl -> tl.searchByAccountFid(taskId, foundFids, tableFileMap, tableMetaDataMap)); + } catch (IOException e) { + throw new InternalException("Failed to read cursor " + e); + } + } else if (searchParameter == SearchParameter.ACCT_NUM) { + if (IndicSearchCommon.isCastableField(getTableName(), "ACCT_NUM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ACCT_NUM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByAcctNumAndMemberCodeFromAccount(searchValue, memberCode)) { + Set foundFids = new HashSet<>(); + tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap)); + accountSubTableLoaders.parallelStream().forEach(tl -> tl.searchByAccountFid(taskId, foundFids, tableFileMap, tableMetaDataMap)); + } catch (IOException e) { + throw new InternalException("Failed to read cursor " + e); + } + } else if (searchParameter == SearchParameter.SERIAL_NUM_ACCT) { + if (IndicSearchCommon.isCastableField(getTableName(), "SERIAL_NUM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("SERIAL_NUM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllBySerialNumAndMemberCodeFromAccount(searchValue, memberCode)) { Set foundFids = new HashSet<>(); tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap)); accountSubTableLoaders.parallelStream().forEach(tl -> tl.searchByAccountFid(taskId, foundFids, tableFileMap, tableMetaDataMap)); diff --git a/src/main/java/ru/nbch/indicsearchservice/loader/IdTableLoader.java b/src/main/java/ru/nbch/indicsearchservice/loader/IdTableLoader.java index 1665f18..b029a66 100644 --- a/src/main/java/ru/nbch/indicsearchservice/loader/IdTableLoader.java +++ b/src/main/java/ru/nbch/indicsearchservice/loader/IdTableLoader.java @@ -16,6 +16,7 @@ import ru.nbch.indicsearchservice.exception.InternalException; import ru.nbch.indicsearchservice.model.TableMetaData; import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.util.IndicSearchCommon; import ru.nbch.indicsearchservice.util.MapperHolder; @Component @@ -66,6 +67,10 @@ public class IdTableLoader implements FirstLevelTableLoader { if (searchParameter == SearchParameter.OGRN || searchParameter == SearchParameter.PASSPORT) { Set foundFids = new HashSet<>(); + if (IndicSearchCommon.isCastableField(getTableName(), "ID_NORM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ID_NORM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } try (Cursor> cursor = searchParameter == SearchParameter.OGRN ? mapperHolder.getIndicMapper().findFidsByOgrnFromId(searchValue) : mapperHolder.getIndicMapper().findFidsByPassportFromId(searchValue)) { @@ -99,6 +104,10 @@ public class IdTableLoader implements FirstLevelTableLoader { if (searchParameter == SearchParameter.OGRN || searchParameter == SearchParameter.PASSPORT) { Set foundFids = new HashSet<>(); + if (IndicSearchCommon.isCastableField(getTableName(), "ID_NORM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ID_NORM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } try (Cursor> cursor = searchParameter == SearchParameter.OGRN ? mapperHolder.getIndicMapper().findFidsByOgrnFromIdForTutdf(searchValue) : mapperHolder.getIndicMapper().findFidsByPassportFromIdForTutdf(searchValue)) { diff --git a/src/main/java/ru/nbch/indicsearchservice/loader/InformationTableLoader.java b/src/main/java/ru/nbch/indicsearchservice/loader/InformationTableLoader.java index 78de216..79eb186 100644 --- a/src/main/java/ru/nbch/indicsearchservice/loader/InformationTableLoader.java +++ b/src/main/java/ru/nbch/indicsearchservice/loader/InformationTableLoader.java @@ -13,6 +13,7 @@ import ru.nbch.indicsearchservice.exception.InternalException; import ru.nbch.indicsearchservice.model.TableMetaData; import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.util.IndicSearchCommon; import ru.nbch.indicsearchservice.util.IndicSearchUtils; import ru.nbch.indicsearchservice.util.MapperHolder; @@ -56,8 +57,22 @@ public class InformationTableLoader implements FirstLevelTableLoader, TableLoade Map tableFileMap, Map tableMetaDataMap) { if (dataType == DataType.TUTDF) { - if (searchParameter == SearchParameter.OLD_APP_NUM || searchParameter == SearchParameter.SERIAL_NUM_APP) { - try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromInformation(memberCode)) { + if (searchParameter == SearchParameter.OLD_APP_NUM) { + if (IndicSearchCommon.isCastableField(getTableName(), "APP_NUM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("APP_NUM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByAppNumAndMemberCodeFromInformation(searchValue, memberCode)) { + tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap)); + } catch (IOException e) { + throw new InternalException("Failed to read cursor " + e); + } + } else if (searchParameter == SearchParameter.SERIAL_NUM_APP) { + if (IndicSearchCommon.isCastableField(getTableName(), "SERIAL_NUM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("SERIAL_NUM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllBySerialNumAndMemberCodeFromInformation(searchValue, memberCode)) { tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap)); } catch (IOException e) { throw new InternalException("Failed to read cursor " + e); diff --git a/src/main/java/ru/nbch/indicsearchservice/loader/RuApplicationTableLoader.java b/src/main/java/ru/nbch/indicsearchservice/loader/RuApplicationTableLoader.java index 4b157ab..5b84bf5 100644 --- a/src/main/java/ru/nbch/indicsearchservice/loader/RuApplicationTableLoader.java +++ b/src/main/java/ru/nbch/indicsearchservice/loader/RuApplicationTableLoader.java @@ -13,6 +13,7 @@ import ru.nbch.indicsearchservice.exception.InternalException; import ru.nbch.indicsearchservice.model.TableMetaData; import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.util.IndicSearchCommon; import ru.nbch.indicsearchservice.util.IndicSearchUtils; import ru.nbch.indicsearchservice.util.MapperHolder; @@ -56,8 +57,22 @@ public class RuApplicationTableLoader implements FirstLevelTableLoader, TableLoa Map tableFileMap, Map tableMetaDataMap) { if (dataType == DataType.GUTDF || dataType == DataType.RUTDF) { - if (searchParameter == SearchParameter.APPLICATION_NUMBER || searchParameter == SearchParameter.OLD_APP_NUM) { - try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromRuApplication(memberCode)) { + if (searchParameter == SearchParameter.APPLICATION_NUMBER) { + if (IndicSearchCommon.isCastableField(getTableName(), "APPLICATION_NUMBER")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("APPLICATION_NUMBER").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByApplicationNumberAndMemberCodeFromRuApplication(searchValue, memberCode)) { + tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap)); + } catch (IOException e) { + throw new InternalException("Failed to read cursor " + e); + } + } else if (searchParameter == SearchParameter.OLD_APP_NUM) { + if (IndicSearchCommon.isCastableField(getTableName(), "OLD_APP_NUM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("OLD_APP_NUM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByOldAppNumAndMemberCodeFromRuApplication(searchValue, memberCode)) { tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, tableMetaDataMap)); } catch (IOException e) { throw new InternalException("Failed to read cursor " + e); diff --git a/src/main/java/ru/nbch/indicsearchservice/loader/RuUidTableLoader.java b/src/main/java/ru/nbch/indicsearchservice/loader/RuUidTableLoader.java index 78e8278..190745c 100644 --- a/src/main/java/ru/nbch/indicsearchservice/loader/RuUidTableLoader.java +++ b/src/main/java/ru/nbch/indicsearchservice/loader/RuUidTableLoader.java @@ -16,6 +16,7 @@ import ru.nbch.indicsearchservice.loader.subtable.RuUidSubTableLoader; import ru.nbch.indicsearchservice.model.TableMetaData; import ru.nbch.indicsearchservice.model.enums.DataType; import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.util.IndicSearchCommon; import ru.nbch.indicsearchservice.util.IndicSearchUtils; import ru.nbch.indicsearchservice.util.MapperHolder; @@ -67,8 +68,24 @@ public class RuUidTableLoader implements FirstLevelTableLoader, TableLoaderForId Map tableFileMap, Map tableMetaDataMap) { if (dataType == DataType.GUTDF || dataType == DataType.RUTDF) { - if (searchParameter == SearchParameter.UUID || searchParameter == SearchParameter.ACCT_NUM) { - try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByMemberCodeFromRuUid(memberCode)) { + if (searchParameter == SearchParameter.UUID) { + if (IndicSearchCommon.isCastableField(getTableName(), "UUID")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("UUID").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByUuidAndMemberCodeFromRuUid(searchValue, memberCode)) { + Set foundFids = new HashSet<>(); + tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap)); + ruUidSubTableLoaders.parallelStream().forEach(tl -> tl.searchByRuUidFid(taskId, foundFids, tableFileMap, tableMetaDataMap)); + } catch (IOException e) { + throw new InternalException("Failed to read cursor " + e); + } + } else if (searchParameter == SearchParameter.ACCT_NUM) { + if (IndicSearchCommon.isCastableField(getTableName(), "ACCT_NUM")) { + int maxSearchValueSize = tableMetaDataMap.get(getTableName()).getFieldMetaData().get("ACCT_NUM").getColumnSize(); + searchValue = searchValue.length() > maxSearchValueSize ? searchValue.substring(0, maxSearchValueSize) : searchValue; + } + try (Cursor> cursor = mapperHolder.getIndicMapper().findAllByAcctNumAndMemberCodeFromRuUid(searchValue, memberCode)) { Set foundFids = new HashSet<>(); tableFileMap.put(getTableName(), indicSearchUtils.writeRowsToFile(taskId, getTableName(), cursor, foundFids, tableMetaDataMap)); ruUidSubTableLoaders.parallelStream().forEach(tl -> tl.searchByRuUidFid(taskId, foundFids, tableFileMap, tableMetaDataMap)); diff --git a/src/main/java/ru/nbch/indicsearchservice/model/TableMetaData.java b/src/main/java/ru/nbch/indicsearchservice/model/TableMetaData.java index 607fea4..31b0470 100644 --- a/src/main/java/ru/nbch/indicsearchservice/model/TableMetaData.java +++ b/src/main/java/ru/nbch/indicsearchservice/model/TableMetaData.java @@ -1,5 +1,7 @@ package ru.nbch.indicsearchservice.model; +import jakarta.annotation.Nullable; +import java.sql.JDBCType; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -43,9 +45,54 @@ public class TableMetaData { ); } - public record FieldMetaData(String fieldName, String fieldLocalization) { + public static class FieldMetaData { + private final String fieldName; + @Nullable + private final String fieldLocalization; + @Nullable + private JDBCType jdbcType; + @Nullable + private Integer columnSize; + + public FieldMetaData(String fieldName, @Nullable String fieldLocalization) { + this.fieldName = fieldName; + this.fieldLocalization = fieldLocalization; + this.jdbcType = null; + this.columnSize = null; + } + public FieldMetaData(String fieldName) { - this(fieldName, null); + this.fieldName = fieldName; + this.fieldLocalization = null; + this.jdbcType = null; + this.columnSize = null; + } + + public String getFieldName() { + return fieldName; + } + + @Nullable + public String getFieldLocalization() { + return fieldLocalization; + } + + @Nullable + public JDBCType getJdbcType() { + return jdbcType; + } + + @Nullable + public Integer getColumnSize() { + return columnSize; + } + + public void setJdbcType(@Nullable JDBCType jdbcType) { + this.jdbcType = jdbcType; + } + + public void setColumnSize(@Nullable Integer columnSize) { + this.columnSize = columnSize; } } } diff --git a/src/main/java/ru/nbch/indicsearchservice/repository/indic/IndicMapper.java b/src/main/java/ru/nbch/indicsearchservice/repository/indic/IndicMapper.java index 27e822b..c3f412e 100644 --- a/src/main/java/ru/nbch/indicsearchservice/repository/indic/IndicMapper.java +++ b/src/main/java/ru/nbch/indicsearchservice/repository/indic/IndicMapper.java @@ -21,8 +21,11 @@ public interface IndicMapper extends MyBatisRepository { Cursor> findAllByFidsFromRuCapability(Set fids); Cursor> findAllByFidsFromRuDebtBurdenInfo(Set fids); - Cursor> findAllByMemberCodeFromRuUid(String memberCode); - Cursor> findAllByMemberCodeFromRuApplication(String memberCode); + Cursor> findAllByAcctNumAndMemberCodeFromRuUid(String acctNum, String memberCode); + Cursor> findAllByUuidAndMemberCodeFromRuUid(String uuid, String memberCode); + Cursor> findAllByApplicationNumberAndMemberCodeFromRuApplication(String applicationNumber, String memberCode); + Cursor> findAllByOldAppNumAndMemberCodeFromRuApplication(String oldAppNum, String memberCode); + Cursor> findFidsByOgrnFromId(String idNorm); Cursor> findFidsByPassportFromId(String idNorm); @@ -68,8 +71,12 @@ public interface IndicMapper extends MyBatisRepository { Cursor> findAllByFidsFromConsumerBankruptcy(Set fids); Cursor> findAllByFidsFromLegal(Set fids); - Cursor> findAllByMemberCodeFromAccount(String memberCode); - Cursor> findAllByMemberCodeFromInformation(String memberCode); + Cursor> findAllByAcctNumAndMemberCodeFromAccount(String acctNum, String memberCode); + Cursor> findAllBySerialNumAndMemberCodeFromAccount(String serialNum, String memberCode); + Cursor> findAllByUuidAndMemberCodeFromAccount(String uuid, String memberCode); + Cursor> findAllBySerialNumAndMemberCodeFromInformation(String serialNum, String memberCode); + Cursor> findAllByAppNumAndMemberCodeFromInformation(String appNum, String memberCode); + Cursor> findFidsByOgrnFromIdForTutdf(String idNorm); Cursor> findFidsByPassportFromIdForTutdf(String idNorm); diff --git a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java index de35147..acb814c 100644 --- a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java +++ b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java @@ -3,18 +3,11 @@ package ru.nbch.indicsearchservice.service; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.Objects; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,15 +22,8 @@ import ru.nbch.indicsearchservice.exception.InternalException; import ru.nbch.indicsearchservice.exporter.ExcelExporter; import ru.nbch.indicsearchservice.exporter.JsonExporter; import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader; -import ru.nbch.indicsearchservice.model.LocalizationObject; -import ru.nbch.indicsearchservice.model.Settings; import ru.nbch.indicsearchservice.model.TableMetaData; -import ru.nbch.indicsearchservice.model.enums.DataType; -import ru.nbch.indicsearchservice.model.enums.SearchParameter; -import ru.nbch.indicsearchservice.repository.bank.LocalizationObjectMapper; -import ru.nbch.indicsearchservice.repository.bank.SettingsMapper; -import ru.nbch.indicsearchservice.util.IndicSearchUtils; -import ru.nbch.indicsearchservice.util.MapperHolder; +import ru.nbch.indicsearchservice.util.TableMetaDataUtil; @Service public class IndicSearchService { @@ -50,29 +36,20 @@ public class IndicSearchService { private final ExcelExporter excelExporter; private final JsonExporter jsonExporter; private final SettingsService settingsService; - private final LocalizationObjectMapper localizationObjectMapper; - private final SettingsMapper settingsMapper; - private final MapperHolder mapperHolder; - private final IndicSearchUtils indicSearchUtils; + private final TableMetaDataUtil tableMetaDataUtil; public IndicSearchService(List tableLoaders, List fileAggregators, ExcelExporter excelExporter, JsonExporter jsonExporter, SettingsService settingsService, - LocalizationObjectMapper localizationObjectMapper, - SettingsMapper settingsMapper, - MapperHolder mapperHolder, - IndicSearchUtils indicSearchUtils) { + TableMetaDataUtil tableMetaDataUtil) { this.tableLoaders = tableLoaders; this.fileAggregators = fileAggregators; this.excelExporter = excelExporter; this.jsonExporter = jsonExporter; this.settingsService = settingsService; - this.localizationObjectMapper = localizationObjectMapper; - this.settingsMapper = settingsMapper; - this.mapperHolder = mapperHolder; - this.indicSearchUtils = indicSearchUtils; + this.tableMetaDataUtil = tableMetaDataUtil; } public AddSearchJobResponseDto addSearchJob(AddSearchJobRequestDto request) { @@ -84,7 +61,7 @@ public class IndicSearchService { log.info("Prep step started. Getting metadata from database"); sw.start(); - Map tableMetaDataMap = prepareTableMetaData(request.getDataType(), request.getSearchParameter()); + Map tableMetaDataMap = tableMetaDataUtil.prepareTableMetaData(request.getDataType(), request.getSearchParameter()); long rowLimit = Long.parseLong(settingsService.getSettingsValueByName("LimitDB")); sw.stop(); @@ -123,6 +100,7 @@ public class IndicSearchService { request.getMemberCode().substring(0, 6), tableFileMap, rowCounter)) + .filter(Objects::nonNull) .findFirst() .orElse(null); @@ -145,7 +123,7 @@ public class IndicSearchService { if (rowCounter.longValue() < rowLimit && !IS_EXPORT_AS_EXCEL) { jsonExporter.writeJsonFile(aggregatedFile, taskId, request.getSearchParameter(), tableMetaDataMap); } else { - fillTableMetaDataWithLocalization(tableMetaDataMap); + tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap); excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue(), tableMetaDataMap); } @@ -170,154 +148,4 @@ public class IndicSearchService { public ResponseEntity export(Long id, String exportType) { return null; } - - private Map prepareTableMetaData(DataType dataType, SearchParameter searchParameter) { - Map tableMetaDataMap = new ConcurrentHashMap<>(); - - // Получаем список таблиц - Set tableNames = tableLoaders.parallelStream() - .map(tl -> tl.getTableNamesIfSuitable(dataType, searchParameter)) - .flatMap(Set::stream) - .collect(Collectors.toSet()); - - if (tableNames.isEmpty()) { - return tableMetaDataMap; - } - - tableNames.forEach(name -> tableMetaDataMap.put(name, new TableMetaData(name))); - - // Составляем массив ключей из SETTINGS - Map keyToTable = new HashMap<>(); - List columnKeys = new ArrayList<>(); - for (String table : tableNames) { - String key = "db.table." + table.toUpperCase() + ".fields"; - columnKeys.add(key); - keyToTable.put(key, table); - } - - // Получаем данные из SETTINGS одним запросом - Map> settingsValues = settingsService.getSettingsValuesByNames(columnKeys); - Set missingTables = new HashSet<>(); - List batchInsertSettings = new ArrayList<>(); - - for (String key : columnKeys) { - String table = keyToTable.get(key); - Optional valueOpt = settingsValues.get(key); - - if (valueOpt.isPresent() && StringUtils.isNotBlank(valueOpt.get())) { - String[] fields = valueOpt.get().split(";"); - for (String field : fields) { - tableMetaDataMap.get(table).addField(field.trim()); - } - } else { - missingTables.add(table); - } - } - - // Если найдены пропущенные значения, то добавляем их в БД - if (!missingTables.isEmpty()) { - List> rows = - mapperHolder.getIndicMapper().getTableColumnsByTableNames(new ArrayList<>(missingTables)); - - Map> tableToColumns = new HashMap<>(); - for (Map row : rows) { - tableToColumns.computeIfAbsent(row.get("TBNAME"), t -> new ArrayList<>()).add(row.get("NAME")); - } - - for (String table : missingTables) { - List columns = tableToColumns.getOrDefault(table, List.of()); - String key = "db.table." + table.toUpperCase() + ".fields"; - String value = String.join(";", columns); - - Settings newSettings = new Settings(); - newSettings.setName(key); - newSettings.setValue(value); - batchInsertSettings.add(newSettings); - - for (String column : columns) { - tableMetaDataMap.get(table).addField(column.trim()); - } - } - - if (!batchInsertSettings.isEmpty()) { - settingsMapper.insertBatch(batchInsertSettings); - } - } - - return tableMetaDataMap; - } - - private void fillTableMetaDataWithLocalization(Map tableMetaDataMap) { - if (tableMetaDataMap == null || tableMetaDataMap.isEmpty()) { - return; - } - - // Получаем все ключи для LOCALIZATION_OBJECT - Set localizationKeys = new HashSet<>(); - for (TableMetaData metaData : tableMetaDataMap.values()) { - String tableUpper = metaData.getTableName().toUpperCase(); - localizationKeys.add("db.viewer.table." + tableUpper); - - for (String field : metaData.getFieldMetaData().keySet()) { - String fieldUpper = field.toUpperCase(); - localizationKeys.add("db.viewer." + tableUpper + ".field." + fieldUpper); - localizationKeys.add("db.viewer.field." + fieldUpper); - } - } - - // Получаем данные из LOCALIZATION_OBJECT одним запросом - Map> localizationValues = indicSearchUtils.getOptionalLocalizationStrings(new ArrayList<>(localizationKeys)); - List toInsertLocalizations = new ArrayList<>(); - - // Вставляем локализацию в tableMetaDataMap - for (TableMetaData metaData : tableMetaDataMap.values()) { - String tableUpper = metaData.getTableName().toUpperCase(); - String tableKey = "db.viewer.table." + tableUpper; - - // Локализация для таблиц - Optional tableLoc = localizationValues.get(tableKey); - if (tableLoc.isPresent() && StringUtils.isNotBlank(tableLoc.get())) { - metaData.setTableNameLocalization(tableLoc.get()); - } else { - String fallbackText = "Таблица " + tableUpper; - metaData.setTableNameLocalization(fallbackText); - - LocalizationObject newLocalizationObject = new LocalizationObject(); - newLocalizationObject.setCode(tableKey); - newLocalizationObject.setText(fallbackText); - toInsertLocalizations.add(newLocalizationObject); - } - - // Локализация для полей - for (String field : metaData.getFieldMetaData().keySet()) { - String fieldUpper = field.toUpperCase(); - String primaryKey = "db.viewer." + tableUpper + ".field." + fieldUpper; - String fallbackKey = "db.viewer.field." + fieldUpper; - - Optional primaryLoc = localizationValues.get(primaryKey); - - if (primaryLoc.isPresent() && StringUtils.isNotBlank(primaryLoc.get())) { - metaData.addLocalization(field, primaryLoc.get()); - } else { - Optional fallbackLoc = localizationValues.get(fallbackKey); - if (fallbackLoc.isPresent() && StringUtils.isNotBlank(fallbackLoc.get())) { - metaData.addLocalization(field, fallbackLoc.get()); - } else { - String fallbackText = "Поле " + fieldUpper; - metaData.addLocalization(field, fallbackText); - - LocalizationObject newLocalizationObject = new LocalizationObject(); - newLocalizationObject.setCode(primaryKey); - newLocalizationObject.setText(fallbackText); - toInsertLocalizations.add(newLocalizationObject); - } - } - } - } - - // Если найдены пропущенные значения, то добавляем их в БД - if (!toInsertLocalizations.isEmpty()) { - localizationObjectMapper.insertBatch(toInsertLocalizations); - } - } } diff --git a/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java b/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java index d16969f..8f1f1a8 100644 --- a/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java +++ b/src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java @@ -2,9 +2,34 @@ package ru.nbch.indicsearchservice.util; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Optional; public class IndicSearchCommon { + private static final Map> CASTABLE_FIELDS = Map.ofEntries( + Map.entry("RU_UID", List.of("ACCT_NUM")), + Map.entry("RU_APPLICATION", List.of("OLD_APP_NUM")), + Map.entry("ID", List.of("ID_NORM")), + Map.entry("ACCOUNT", List.of("ACCT_NUM")), + Map.entry("INFORMATION", List.of("APP_NUM")) + ); + + public static Map> getCastableFields() { + return CASTABLE_FIELDS; + } + + public static boolean isCastableField(String tableName, String fieldName) { + if (tableName == null || tableName.isEmpty() || fieldName == null || fieldName.isEmpty()) { + return false; + } + + if (CASTABLE_FIELDS.containsKey(tableName.toUpperCase())) { + return CASTABLE_FIELDS.get(tableName).contains(fieldName.toUpperCase()); + } + + return false; + } + public static Optional getRowValue(String string, String key) { return getRowValue(Arrays.asList(string.split("%%%", -1)), key); } diff --git a/src/main/java/ru/nbch/indicsearchservice/util/TableMetaDataUtil.java b/src/main/java/ru/nbch/indicsearchservice/util/TableMetaDataUtil.java new file mode 100644 index 0000000..1c9c297 --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/util/TableMetaDataUtil.java @@ -0,0 +1,233 @@ +package ru.nbch.indicsearchservice.util; + +import java.sql.Connection; +import java.sql.JDBCType; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.datasource.DataSourceUtils; +import org.springframework.stereotype.Component; +import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader; +import ru.nbch.indicsearchservice.model.LocalizationObject; +import ru.nbch.indicsearchservice.model.Settings; +import ru.nbch.indicsearchservice.model.TableMetaData; +import ru.nbch.indicsearchservice.model.enums.DataType; +import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.repository.bank.LocalizationObjectMapper; +import ru.nbch.indicsearchservice.repository.bank.SettingsMapper; +import ru.nbch.indicsearchservice.service.SettingsService; + +@Component +public class TableMetaDataUtil { + private final List tableLoaders; + private final SettingsService settingsService; + private final MapperHolder mapperHolder; + private final SettingsMapper settingsMapper; + private final LocalizationObjectMapper localizationObjectMapper; + private final IndicSearchUtils indicSearchUtils; + private final DataSource indicDataSource; + + public TableMetaDataUtil(List tableLoaders, + SettingsService settingsService, + MapperHolder mapperHolder, + SettingsMapper settingsMapper, + LocalizationObjectMapper localizationObjectMapper, + IndicSearchUtils indicSearchUtils, + @Qualifier("indicDataSource") DataSource indicDataSource) { + this.tableLoaders = tableLoaders; + this.settingsService = settingsService; + this.mapperHolder = mapperHolder; + this.settingsMapper = settingsMapper; + this.localizationObjectMapper = localizationObjectMapper; + this.indicSearchUtils = indicSearchUtils; + this.indicDataSource = indicDataSource; + } + + public Map prepareTableMetaData(DataType dataType, SearchParameter searchParameter) { + Map tableMetaDataMap = new ConcurrentHashMap<>(); + + // Получаем список таблиц + Set tableNames = tableLoaders.parallelStream() + .map(tl -> tl.getTableNamesIfSuitable(dataType, searchParameter)) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + + if (tableNames.isEmpty()) { + return tableMetaDataMap; + } + + tableNames.forEach(name -> tableMetaDataMap.put(name, new TableMetaData(name))); + + // Составляем массив ключей из SETTINGS + Map keyToTable = new HashMap<>(); + List columnKeys = new ArrayList<>(); + for (String table : tableNames) { + String key = "db.table." + table.toUpperCase() + ".fields"; + columnKeys.add(key); + keyToTable.put(key, table); + } + + // Получаем данные из SETTINGS одним запросом + Map> settingsValues = settingsService.getSettingsValuesByNames(columnKeys); + Set missingTables = new HashSet<>(); + List batchInsertSettings = new ArrayList<>(); + + for (String key : columnKeys) { + String table = keyToTable.get(key); + Optional valueOpt = settingsValues.get(key); + + if (valueOpt.isPresent() && StringUtils.isNotBlank(valueOpt.get())) { + String[] fields = valueOpt.get().split(";"); + for (String field : fields) { + tableMetaDataMap.get(table).addField(field.trim()); + } + } else { + missingTables.add(table); + } + } + + // Если найдены пропущенные значения, то добавляем их в БД + if (!missingTables.isEmpty()) { + List> rows = + mapperHolder.getIndicMapper().getTableColumnsByTableNames(new ArrayList<>(missingTables)); + + Map> tableToColumns = new HashMap<>(); + for (Map row : rows) { + tableToColumns.computeIfAbsent(row.get("TBNAME"), t -> new ArrayList<>()).add(row.get("NAME")); + } + + for (String table : missingTables) { + List columns = tableToColumns.getOrDefault(table, List.of()); + String key = "db.table." + table.toUpperCase() + ".fields"; + String value = String.join(";", columns); + + Settings newSettings = new Settings(); + newSettings.setName(key); + newSettings.setValue(value); + batchInsertSettings.add(newSettings); + + for (String column : columns) { + tableMetaDataMap.get(table).addField(column.trim()); + } + } + + if (!batchInsertSettings.isEmpty()) { + settingsMapper.insertBatch(batchInsertSettings); + } + } + + // Получаем размеры некоторых полей + for (Map.Entry> entry : IndicSearchCommon.getCastableFields().entrySet()) { + String tableName = entry.getKey(); + List tableFields = entry.getValue(); + + if (!tableMetaDataMap.containsKey(tableName)) { + continue; + } + + try (Connection connection = DataSourceUtils.getConnection(indicDataSource); + ResultSet resultSet = connection.getMetaData().getColumns(null, null, tableName, null)) { + while (resultSet.next()) { + String columnName = resultSet.getString("COLUMN_NAME"); + if (tableFields.contains(columnName) && tableMetaDataMap.get(tableName).getFieldMetaData().containsKey(columnName)) { + TableMetaData.FieldMetaData foundFieldMetaData = tableMetaDataMap.get(tableName).getFieldMetaData().get(columnName); + foundFieldMetaData.setJdbcType(JDBCType.valueOf(resultSet.getInt("DATA_TYPE"))); + foundFieldMetaData.setColumnSize(resultSet.getInt("COLUMN_SIZE")); + } + } + + DataSourceUtils.releaseConnection(connection, indicDataSource); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + return tableMetaDataMap; + } + + public void fillTableMetaDataWithLocalization(Map tableMetaDataMap) { + if (tableMetaDataMap == null || tableMetaDataMap.isEmpty()) { + return; + } + + // Получаем все ключи для LOCALIZATION_OBJECT + Set localizationKeys = new HashSet<>(); + for (TableMetaData metaData : tableMetaDataMap.values()) { + String tableUpper = metaData.getTableName().toUpperCase(); + localizationKeys.add("db.viewer.table." + tableUpper); + + for (String field : metaData.getFieldMetaData().keySet()) { + String fieldUpper = field.toUpperCase(); + localizationKeys.add("db.viewer." + tableUpper + ".field." + fieldUpper); + localizationKeys.add("db.viewer.field." + fieldUpper); + } + } + + // Получаем данные из LOCALIZATION_OBJECT одним запросом + Map> localizationValues = indicSearchUtils.getOptionalLocalizationStrings(new ArrayList<>(localizationKeys)); + List toInsertLocalizations = new ArrayList<>(); + + // Вставляем локализацию в tableMetaDataMap + for (TableMetaData metaData : tableMetaDataMap.values()) { + String tableUpper = metaData.getTableName().toUpperCase(); + String tableKey = "db.viewer.table." + tableUpper; + + // Локализация для таблиц + Optional tableLoc = localizationValues.get(tableKey); + if (tableLoc.isPresent() && StringUtils.isNotBlank(tableLoc.get())) { + metaData.setTableNameLocalization(tableLoc.get()); + } else { + String fallbackText = "Таблица " + tableUpper; + metaData.setTableNameLocalization(fallbackText); + + LocalizationObject newLocalizationObject = new LocalizationObject(); + newLocalizationObject.setCode(tableKey); + newLocalizationObject.setText(fallbackText); + toInsertLocalizations.add(newLocalizationObject); + } + + // Локализация для полей + for (String field : metaData.getFieldMetaData().keySet()) { + String fieldUpper = field.toUpperCase(); + String primaryKey = "db.viewer." + tableUpper + ".field." + fieldUpper; + String fallbackKey = "db.viewer.field." + fieldUpper; + + Optional primaryLoc = localizationValues.get(primaryKey); + + if (primaryLoc.isPresent() && StringUtils.isNotBlank(primaryLoc.get())) { + metaData.addLocalization(field, primaryLoc.get()); + } else { + Optional fallbackLoc = localizationValues.get(fallbackKey); + if (fallbackLoc.isPresent() && StringUtils.isNotBlank(fallbackLoc.get())) { + metaData.addLocalization(field, fallbackLoc.get()); + } else { + String fallbackText = "Поле " + fieldUpper; + metaData.addLocalization(field, fallbackText); + + LocalizationObject newLocalizationObject = new LocalizationObject(); + newLocalizationObject.setCode(primaryKey); + newLocalizationObject.setText(fallbackText); + toInsertLocalizations.add(newLocalizationObject); + } + } + } + } + + // Если найдены пропущенные значения, то добавляем их в БД + if (!toInsertLocalizations.isEmpty()) { + localizationObjectMapper.insertBatch(toInsertLocalizations); + } + } +} diff --git a/src/main/resources/mappers/indic/indic_mapper.xml b/src/main/resources/mappers/indic/indic_mapper.xml index 72c8273..f6a91a4 100644 --- a/src/main/resources/mappers/indic/indic_mapper.xml +++ b/src/main/resources/mappers/indic/indic_mapper.xml @@ -102,30 +102,52 @@ WITH UR - SELECT * FROM INDIC.RU_UID - WHERE (MEMBER_CODE LIKE #{memberCode} || '%' + WHERE ACCT_NUM = #{acctNum} + AND (MEMBER_CODE LIKE #{memberCode} || '%' OR REAL_MEMBERCODE LIKE #{memberCode} || '%') ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC WITH UR - + SELECT * + FROM INDIC.RU_UID + WHERE UUID = #{uuid} + AND (MEMBER_CODE LIKE #{memberCode} || '%' + OR REAL_MEMBERCODE LIKE #{memberCode} || '%') + ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC + WITH UR + + + + + SELECT FID FROM INDIC.ID - WHERE CAST(ID_NORM AS VARCHAR(2000)) = #{idNorm} + WHERE ID_NORM = #{idNorm} AND (ID_TYPE LIKE '21' OR ID_TYPE2 LIKE '21' OR ID_TYPE LIKE '10' OR ID_TYPE2 LIKE '31') ORDER BY LAST_UPDATED_DT DESC WITH UR @@ -578,28 +600,57 @@ WITH UR - SELECT * FROM INDIC.ACCOUNT - WHERE MEMBER_CODE LIKE #{memberCode} || '%' + WHERE ACCT_NUM = #{acctNum} + AND MEMBER_CODE LIKE #{memberCode} || '%' ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC WITH UR - + SELECT * + FROM INDIC.ACCOUNT + WHERE SERIAL_NUM = #{serialNum} + AND MEMBER_CODE LIKE #{memberCode} || '%' + ORDER BY SERIAL_NUM, LAST_UPDATED_DT DESC + WITH UR + + + + + + + SELECT FID FROM INDIC.ID - WHERE CAST(ID_NORM AS VARCHAR(2000)) = #{idNorm} + WHERE ID_NORM = #{idNorm} AND ID_TYPE in ('10', '21') ORDER BY LAST_UPDATED_DT DESC WITH UR diff --git a/src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java b/src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java index f2562b2..ed816a5 100644 --- a/src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java +++ b/src/test/java/ru/nbch/indicsearchservice/exporter/JsonExporterTest.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -19,13 +20,18 @@ import ru.nbch.indicsearchservice.model.enums.SearchParameter; @SpringBootTest class JsonExporterTest { @Autowired - ApplicationSettings applicationSettings; + ApplicationSettings settings; @Autowired JsonExporter jsonExporter; + @BeforeEach + void setUp() { + settings.setStorageDirectory(Path.of("src/test/resources/storage").toFile()); + } + @Test void writeJsonFile_shouldCreateJsonFilePASSPORT() throws IOException { - Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv"); + Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("test-json-aggregated.csv"); UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907"); TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID"); @@ -41,16 +47,18 @@ class JsonExporterTest { jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.PASSPORT, tableMetaDataMap); - Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json"); + Path resultFile = settings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json"); List resultFileStrings = Files.readAllLines(resultFile); + Files.deleteIfExists(resultFile); + assertThat(resultFileStrings.getFirst()) .isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]},{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}"); } @Test void writeJsonFile_shouldCreateJsonFileUUID() throws IOException { - Path aggregatedFile = Path.of("src/test/resources/storage/test-json-aggregated.csv"); + Path aggregatedFile = settings.getStorageDirectory().toPath().resolve("test-json-aggregated.csv"); UUID uuid = UUID.fromString("c3805074-e840-4d4e-bfa3-eee1fefd4907"); TableMetaData ruUidTableTableMetaData = new TableMetaData("RU_UID"); @@ -66,9 +74,11 @@ class JsonExporterTest { jsonExporter.writeJsonFile(aggregatedFile, uuid, SearchParameter.UUID, tableMetaDataMap); - Path resultFile = applicationSettings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json"); + Path resultFile = settings.getStorageDirectory().toPath().resolve("c3805074-e840-4d4e-bfa3-eee1fefd4907.json"); List resultFileStrings = Files.readAllLines(resultFile); + Files.deleteIfExists(resultFile); + assertThat(resultFileStrings.getFirst()) .isEqualTo("{\"datasheets\":[[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"11\"}],\"related_data\":[]}],[{\"table_name\":\"RU_UID\",\"field_data\":[{\"key\":\"FID\",\"value\":\"1\"},{\"key\":\"SERIAL_NUM\",\"value\":\"21\"}],\"related_data\":[{\"table_name\":\"RU_TRADE\",\"field_data\":[],\"related_data\":[]}]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"2\"},{\"key\":\"SERIAL_NUM\",\"value\":\"12\"}],\"related_data\":[]}],[{\"table_name\":\"RU_APPLICATION\",\"field_data\":[{\"key\":\"FID\",\"value\":\"3\"},{\"key\":\"SERIAL_NUM\",\"value\":\"22\"}],\"related_data\":[]}]]}"); }