From d6992e00c0f1c585cd2106f5eb188674e0934337 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Thu, 26 Jun 2025 16:28:57 +0300 Subject: [PATCH] Optimize fids loading --- .../mapper/indic/PhoneMapper.java | 5 +- .../credit_tracker/service/SignalService.java | 7 ++- .../service/indic/FlagsService.java | 7 ++- .../service/indic/PhoneService.java | 57 +++++++++++++++---- .../mybatis/mapper/indic/PhoneMapper.xml | 16 ++++-- 5 files changed, 74 insertions(+), 18 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/indic/PhoneMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/indic/PhoneMapper.java index cf5f797..791a4c6 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/indic/PhoneMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/indic/PhoneMapper.java @@ -2,8 +2,11 @@ package ru.nbch.credit_tracker.mapper.indic; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; +import ru.nbch.credit_tracker.service.indic.PhoneService; + +import java.util.List; @Mapper public interface PhoneMapper { - Integer findFidByNumber(@Param("phone") String phone); + List findFids(@Param("phones") List phones); } diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index bb729ca..729118b 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -57,7 +57,12 @@ public class SignalService { public Integer registerPackage(MonitoringRequest request) { Integer packageId = mapper.registerPackage(request, LocalDateTime.now(), Status.PROCESSING.name()); - mapper.registerSubjects(request.getSubjects().getSubject(), packageId); + int batchSize = 10000; + List subjects = request.getSubjects().getSubject(); + for (int i = 0; i < subjects.size(); i += batchSize) { + int end = Math.min(i + batchSize, subjects.size()); + mapper.registerSubjects(subjects.subList(i, end), packageId); + } return packageId; } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 6401f6f..7a26b0d 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -28,6 +28,7 @@ public class FlagsService { flagsData.setFlag90days(calcFlag90Days(fid)); return flagsData; } + /** * По FID из RU_PASTDUEARREAR выбираются все уникальные ACC_SERIAL_NUM с максимальной датой REPORTED_DT. * Если по хотя бы одной такой записи AMT_PAST_DUE > 5000, то флаг устанавливается в 1. @@ -45,7 +46,11 @@ public class FlagsService { .map(Optional::get) .toList(); - if (!dataList.isEmpty() && dataList.stream().anyMatch(item -> item != null && item.getAmountPastDue() != null && item.getAmountPastDue().compareTo(BigDecimal.valueOf(5000)) > 0)) { + if (!dataList.isEmpty() && dataList.stream() + .anyMatch(item -> + item != null && item.getAmountPastDue() != null && + item.getAmountPastDue().compareTo(BigDecimal.valueOf(5000)) > 0) + ) { return 1; } else { return 0; diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java index f653e17..b138846 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java @@ -7,8 +7,9 @@ import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.entities.dto.SubjectData; import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; -import java.util.ArrayList; -import java.util.List; +import java.time.LocalDate; +import java.util.*; +import java.util.stream.Collectors; @Service public class PhoneService { @@ -25,18 +26,44 @@ public class PhoneService { public List findFidsPhoneData(List subjects, boolean processNullFids) { List fidsPhoneDataList = new ArrayList<>(); - subjects.forEach(subject -> { - String phone = "9" + subject.getPhone(); - Integer fid = mapper.findFidByNumber(phone); - if (fid != null) { - fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), fid, phone, subject.getSubjectId())); - } else if (processNullFids) { - fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), null, phone, subject.getSubjectId())); - } - }); + int batchSize = 10000; + for (int i = 0; i < subjects.size(); i += batchSize) { + int end = Math.min(i + batchSize, subjects.size()); + List currentSubjects = subjects.subList(i, end); + List fidsData = mapper.findFids(currentSubjects.stream().map(subject -> "9" + subject.getPhone()).toList()); + + Map fidsMap = convertFidsDataToMap(fidsData); + + currentSubjects.forEach(subject -> { + String phone = "9" + subject.getPhone(); + Integer fid = fidsMap.get(phone); + if (fid != null) { + fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), fid, phone, subject.getSubjectId())); + } else if (processNullFids) { + fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), null, phone, subject.getSubjectId())); + } + }); + } return fidsPhoneDataList; } + private Map convertFidsDataToMap(List fidsData) { + if (fidsData == null || fidsData.isEmpty()) { + return new HashMap<>(); + } + return fidsData.stream() + .collect(Collectors.groupingBy( + FidsData::getPhone, + Collectors.collectingAndThen( + Collectors.maxBy( + Comparator.comparing(FidsData::getFileSinceDate) + .thenComparing(FidsData::getFid) + ), + data -> data.map(FidsData::getFid).orElse(null) + ) + )); + } + @Getter @Setter @AllArgsConstructor @@ -46,4 +73,12 @@ public class PhoneService { private String phone; private String subjectId; } + + @Getter + @Setter + public static class FidsData { + private Integer fid; + private String phone; + private LocalDate fileSinceDate; + } } diff --git a/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml b/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml index 068bed1..4c8083d 100644 --- a/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml @@ -5,11 +5,19 @@ - + SELECT NUMBER, FID, FILE_SINCE_DT FROM PHONE_NORM - WHERE NUMBER = #{phone} + WHERE NUMBER in + + #{phone} + ORDER BY FILE_SINCE_DT DESC, FID DESC - LIMIT 1 \ No newline at end of file