treshenkov/indic-search-service
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

csv separator changed to %%%

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2026-05-22 16:47:30 +03:00
parent 74c5c116cf
commit 0f431b54c1
5 changed files with 20 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/main/java/ru/nbch/indicsearchservice/aggregator/FileAggregatorCommon.java
View file

@ -68,7 +68,7 @@ public class FileAggregatorCommon {
String ruUidLine; String ruUidLine;
while ((ruUidLine = ruUidFileReader.readLine()) != null) { while ((ruUidLine = ruUidFileReader.readLine()) != null) {
List<String> lineList = Arrays.asList(ruUidLine.split(";")); List<String> lineList = Arrays.asList(ruUidLine.split("%%%"));
Optional<String> foundMemberCode = IndicSearchCommon.getRowValue(lineList, "MEMBER_CODE"); Optional<String> foundMemberCode = IndicSearchCommon.getRowValue(lineList, "MEMBER_CODE");
if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) { if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) {
@ -94,7 +94,7 @@ public class FileAggregatorCommon {
continue; continue;
} }
List<String> childLineList = Arrays.asList(childLine.split(";")); List<String> childLineList = Arrays.asList(childLine.split("%%%"));
Optional<String> foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM"); Optional<String> foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM");
if (foundChildAccSerialNum.isEmpty()) { if (foundChildAccSerialNum.isEmpty()) {
throw new RuntimeException("ACC_SERIAL_NUM was not found!"); throw new RuntimeException("ACC_SERIAL_NUM was not found!");
@ -103,7 +103,7 @@ public class FileAggregatorCommon {
if (ruUidSerialNum > childAccSerialNum) { if (ruUidSerialNum > childAccSerialNum) {
while ((childLine = childReader.nextLine()) != null) { while ((childLine = childReader.nextLine()) != null) {
childLineList = Arrays.asList(childLine.split(";")); childLineList = Arrays.asList(childLine.split("%%%"));
foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM"); foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM");
if (foundChildAccSerialNum.isEmpty()) { if (foundChildAccSerialNum.isEmpty()) {
throw new RuntimeException("ACC_SERIAL_NUM was not found!"); throw new RuntimeException("ACC_SERIAL_NUM was not found!");
@ -134,7 +134,7 @@ public class FileAggregatorCommon {
} }
while ((childLine = childReader.nextLine()) != null) { while ((childLine = childReader.nextLine()) != null) {
childLineList = Arrays.asList(childLine.split(";")); childLineList = Arrays.asList(childLine.split("%%%"));
foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM"); foundChildAccSerialNum = IndicSearchCommon.getRowValue(childLineList, "ACC_SERIAL_NUM");
if (foundChildAccSerialNum.isEmpty()) { if (foundChildAccSerialNum.isEmpty()) {
throw new RuntimeException("ACC_SERIAL_NUM was not found!"); throw new RuntimeException("ACC_SERIAL_NUM was not found!");
@ -172,7 +172,7 @@ public class FileAggregatorCommon {
MovableBufferedReader blocksFileReader = new MovableBufferedReader(Files.newBufferedReader(tableFileMap.get(parentTableName + "_BLOCKS")))) { MovableBufferedReader blocksFileReader = new MovableBufferedReader(Files.newBufferedReader(tableFileMap.get(parentTableName + "_BLOCKS")))) {
String parentLine; String parentLine;
while ((parentLine = parentFileReader.readLine()) != null) { while ((parentLine = parentFileReader.readLine()) != null) {
List<String> parentLineList = Arrays.asList(parentLine.split(";")); List<String> parentLineList = Arrays.asList(parentLine.split("%%%"));
Optional<String> foundMemberCode = IndicSearchCommon.getRowValue(parentLineList, "MEMBER_CODE"); Optional<String> foundMemberCode = IndicSearchCommon.getRowValue(parentLineList, "MEMBER_CODE");
if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) { if (foundMemberCode.isPresent() && !foundMemberCode.get().startsWith(memberCode)) {
@ -194,7 +194,7 @@ public class FileAggregatorCommon {
continue; continue;
} }
List<String> childLineList = Arrays.asList(blocksLine.split(";")); List<String> childLineList = Arrays.asList(blocksLine.split("%%%"));
Optional<String> foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM"); Optional<String> foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM");
if (foundChildParentSerialNum.isEmpty()) { if (foundChildParentSerialNum.isEmpty()) {
throw new RuntimeException("PARENT_SERIAL_NUM was not found!"); throw new RuntimeException("PARENT_SERIAL_NUM was not found!");
@ -203,7 +203,7 @@ public class FileAggregatorCommon {
if (parentSerialNum > blocksParentSerialNum) { if (parentSerialNum > blocksParentSerialNum) {
while ((blocksLine = blocksFileReader.nextLine()) != null) { while ((blocksLine = blocksFileReader.nextLine()) != null) {
childLineList = Arrays.asList(blocksLine.split(";")); childLineList = Arrays.asList(blocksLine.split("%%%"));
foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM"); foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM");
if (foundChildParentSerialNum.isEmpty()) { if (foundChildParentSerialNum.isEmpty()) {
throw new RuntimeException("PARENT_SERIAL_NUM was not found!"); throw new RuntimeException("PARENT_SERIAL_NUM was not found!");
@ -226,7 +226,7 @@ public class FileAggregatorCommon {
rowCounter.increment(); rowCounter.increment();
while ((blocksLine = blocksFileReader.nextLine()) != null) { while ((blocksLine = blocksFileReader.nextLine()) != null) {
childLineList = Arrays.asList(blocksLine.split(";")); childLineList = Arrays.asList(blocksLine.split("%%%"));
foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM"); foundChildParentSerialNum = IndicSearchCommon.getRowValue(childLineList, "PARENT_SERIAL_NUM");
if (foundChildParentSerialNum.isEmpty()) { if (foundChildParentSerialNum.isEmpty()) {
throw new RuntimeException("PARENT_SERIAL_NUM was not found!"); throw new RuntimeException("PARENT_SERIAL_NUM was not found!");

10
src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java
View file

@ -62,12 +62,12 @@ public class ExcelExporter implements Exporter {
List<String> rows = new ArrayList<>(); List<String> rows = new ArrayList<>();
while (iterator.hasNext()) { while (iterator.hasNext()) {
String line = iterator.next(); String line = iterator.next();
List<String> lineList = Arrays.asList(line.split(";")); List<String> lineList = Arrays.asList(line.split("%%%"));
log.info("Processing row: {} / {}", count++, totalRows); log.info("Processing row: {} / {}", count++, totalRows);
if (!isFirst && (FIRST_LEVEL_TABLES.contains(lineList.getFirst()) || line.equals("EOF"))) { if (!isFirst && (FIRST_LEVEL_TABLES.contains(lineList.getFirst()) || line.equals("EOF"))) {
lastTableName = rows.getFirst().split(";")[0]; lastTableName = rows.getFirst().split("%%%")[0];
writeToExcel(workbook, rows, sheetRowCounter, sheetTableCounter, headerCellStyle, cellStyle); writeToExcel(workbook, rows, sheetRowCounter, sheetTableCounter, headerCellStyle, cellStyle);
if (lastTableName != null && !lastTableName.equals(lineList.getFirst())) { if (lastTableName != null && !lastTableName.equals(lineList.getFirst())) {
sheetRowCounter.setValue(0L); sheetRowCounter.setValue(0L);
@ -96,10 +96,10 @@ public class ExcelExporter implements Exporter {
return; return;
} }
String parentTableName = rows.getFirst().split(";")[0]; String parentTableName = rows.getFirst().split("%%%")[0];
Sheet tableSheet; Sheet tableSheet;
int rowsApproximateSize = rows.stream().mapToInt(r -> r.split(";", -1).length).sum(); int rowsApproximateSize = rows.stream().mapToInt(r -> r.split("%%%", -1).length).sum();
if (sheetRowCounter.longValue() + (rowsApproximateSize * 2L) >= EXCEL_SHEET_MAX_SIZE) { if (sheetRowCounter.longValue() + (rowsApproximateSize * 2L) >= EXCEL_SHEET_MAX_SIZE) {
tableSheet = createSheet(workbook, parentTableName.toUpperCase() + "_" + sheetTableCounter.incrementAndGet()); tableSheet = createSheet(workbook, parentTableName.toUpperCase() + "_" + sheetTableCounter.incrementAndGet());
sheetRowCounter.setValue(0L); sheetRowCounter.setValue(0L);
@ -108,7 +108,7 @@ public class ExcelExporter implements Exporter {
} }
for (String row : rows) { for (String row : rows) {
List<String> rowList = Arrays.asList(row.split(";", -1)); List<String> rowList = Arrays.asList(row.split("%%%", -1));
String tableName = rowList.getFirst(); String tableName = rowList.getFirst();
List<String> rowSublist = rowList.subList(1, rowList.size()); List<String> rowSublist = rowList.subList(1, rowList.size());

4
src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java
View file

@ -48,7 +48,7 @@ public class JsonExporter implements Exporter {
continue; continue;
} }
List<String> lineList = Arrays.asList(line.split(";", -1)); List<String> lineList = Arrays.asList(line.split("%%%", -1));
String tableName = lineList.getFirst(); String tableName = lineList.getFirst();
if (!isFirst && (FIRST_LEVEL_TABLES.contains(tableName) || line.equals("EOF"))) { if (!isFirst && (FIRST_LEVEL_TABLES.contains(tableName) || line.equals("EOF"))) {
@ -90,7 +90,7 @@ public class JsonExporter implements Exporter {
} }
private TableData parseSingleRow(String row) { private TableData parseSingleRow(String row) {
String[] parts = row.split(";", -1); String[] parts = row.split("%%%", -1);
if (parts.length == 0 || parts[0].isEmpty()) { if (parts.length == 0 || parts[0].isEmpty()) {
return null; return null;

2
src/main/java/ru/nbch/indicsearchservice/util/IndicSearchCommon.java
View file

@ -6,7 +6,7 @@ import java.util.Optional;
public class IndicSearchCommon { public class IndicSearchCommon {
public static Optional<String> getRowValue(String string, String key) { public static Optional<String> getRowValue(String string, String key) {
return getRowValue(Arrays.asList(string.split(";", -1)), key); return getRowValue(Arrays.asList(string.split("%%%", -1)), key);
} }
public static Optional<String> getRowValue(List<String> stringValues, String key) { public static Optional<String> getRowValue(List<String> stringValues, String key) {

8
src/main/java/ru/nbch/indicsearchservice/util/IndicSearchUtils.java
View file

@ -35,18 +35,18 @@ public class IndicSearchUtils {
Cursor<LinkedHashMap<String, Object>> cursor = mapperFunc.apply(fids)) { Cursor<LinkedHashMap<String, Object>> cursor = mapperFunc.apply(fids)) {
for (LinkedHashMap<String, Object> row : cursor) { for (LinkedHashMap<String, Object> row : cursor) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append(tableName).append(";"); sb.append(tableName).append("%%%");
for (Map.Entry<String, Object> entry : row.entrySet()) { for (Map.Entry<String, Object> entry : row.entrySet()) {
String columnName = entry.getKey(); String columnName = entry.getKey();
Object columnValue = entry.getValue(); Object columnValue = entry.getValue();
sb.append(columnName.toUpperCase()).append(";"); sb.append(columnName.toUpperCase()).append("%%%");
if (columnValue != null) { if (columnValue != null) {
sb.append(columnValue).append(";"); sb.append(columnValue).append("%%%");
} else { } else {
sb.append(";"); sb.append("%%%");
} }
} }