From 4af0bad507e4b5d70d2acb6a8674d97eb1c870c6 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Mon, 30 Jun 2025 12:17:00 +0300 Subject: [PATCH] Use batchs for finding fids and flags --- .../entities/dto/SubjectData.java | 4 + .../mapper/indic/FlagsMapper.java | 7 +- .../mapper/signal/SignalMapper.java | 2 + .../credit_tracker/service/SignalService.java | 55 ++++++--- .../service/UpdatingFlagsService.java | 2 +- .../service/indic/FlagsService.java | 109 +++++++++++------- .../service/indic/PhoneService.java | 7 +- .../mybatis/mapper/indic/FlagsMapper.xml | 79 ++++++++++--- .../mybatis/mapper/indic/PhoneMapper.xml | 4 +- .../mybatis/mapper/signal/SignalMapper.xml | 85 ++++++++++---- 10 files changed, 244 insertions(+), 110 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/entities/dto/SubjectData.java b/src/main/java/ru/nbch/credit_tracker/entities/dto/SubjectData.java index f3cf750..95e1cd9 100644 --- a/src/main/java/ru/nbch/credit_tracker/entities/dto/SubjectData.java +++ b/src/main/java/ru/nbch/credit_tracker/entities/dto/SubjectData.java @@ -1,12 +1,16 @@ package ru.nbch.credit_tracker.entities.dto; +import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import java.time.LocalDateTime; @Getter @Setter +@AllArgsConstructor +@NoArgsConstructor public class SubjectData { private Integer id; private Integer packageId; diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java index 3baaa97..c64cd90 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java @@ -10,9 +10,10 @@ import java.util.List; @Mapper public interface FlagsMapper { - List findFlagsDebt5000(@Param("fid") Integer fid); + List findFlagsDebt5000(@Param("fids") List fids); - boolean findFlag90ByPastDueArrear(@Param("fid") Integer fid, @Param("reportedDate") LocalDate reportedDate); + List findFlag90ByPastDueArrear(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); + + List findFlag90ByRuPaymt(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); - boolean findFlag90ByRuPaymt(@Param("fid") Integer fid, @Param("reportedDate") LocalDate reportedDate); } diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/signal/SignalMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/signal/SignalMapper.java index eafd1aa..a61ccf8 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/signal/SignalMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/signal/SignalMapper.java @@ -42,5 +42,7 @@ public interface SignalMapper { void setHits(@Param("hits") List hits); + void updateSubjects(@Param("subjects") List subjects); + void setPackageLastDownloadedReport(@Param("id") Integer id, @Param("fileName") String fileName); } diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index 3911df3..3a98082 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -1,5 +1,6 @@ package ru.nbch.credit_tracker.service; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; @@ -26,8 +27,12 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Objects; +@Slf4j @Service public class SignalService { @@ -39,6 +44,8 @@ public class SignalService { private final FlagsService flagsService; private final PersonService personService; + private final int batchSize = 10000; + public SignalService(CreditTrackerSettings config, SignalMapper mapper, PhoneService phoneService, @@ -58,7 +65,6 @@ public class SignalService { public Integer registerPackage(MonitoringRequest request) { Integer packageId = mapper.registerPackage(request, LocalDateTime.now(), Status.PROCESSING.name()); - int batchSize = 10000; List subjects = request.getSubjects().getSubject(); for (int i = 0; i < subjects.size(); i += batchSize) { int end = Math.min(i + batchSize, subjects.size()); @@ -69,7 +75,37 @@ public class SignalService { public List registerSubjectsFidAndDebts(Integer packageId) { List subjects = mapper.findSubjectsData(packageId); - return updateFlags(subjects); + return updateFlags(subjects, packageId); + } + + public List updateFlags(List subjects, Integer packageId) { + List fidsPhoneDataList = phoneService.findFidsPhoneData(subjects, true); + LocalDateTime calculatedAt = LocalDateTime.now(); + for (int i = 0; i < fidsPhoneDataList.size(); i += batchSize) { + int end = Math.min(i + batchSize, fidsPhoneDataList.size()); + List currentList = fidsPhoneDataList.subList(i, end); + Map flagsDataMap = flagsService.findFlags(currentList.stream().map(PhoneService.FidsPhoneData::getFid).filter(Objects::nonNull).toList()); + List subjectToUpdate = new ArrayList<>(); + + currentList.forEach(data -> { + SubjectData subject; + if (data.getFid() != null) { + FlagsService.FlagsData flagsData = flagsDataMap.get(data.getFid()); + subject = new SubjectData(data.getId(), packageId, data.getSubjectId(), data.getPhone(), data.getFid(), flagsData.getFlagDebt5000(), flagsData.getFlag90days(), calculatedAt); + } else { + subject = new SubjectData(data.getId(), packageId, data.getSubjectId(), data.getPhone(),null, null, null, calculatedAt); + } + subjectToUpdate.add(subject); + }); + + int updateBatchSize = batchSize / 8; // 8 параметров на одну запись для обновления. Максимальное количество параметров ~65000 + for (int j = 0; j < subjectToUpdate.size(); j += updateBatchSize) { + int endIndex = Math.min(j + updateBatchSize, subjectToUpdate.size()); + mapper.updateSubjects(subjectToUpdate.subList(j, endIndex)); + } + } + + return fidsPhoneDataList; } public void rejectPackage(MonitoringRequest request) { @@ -196,19 +232,4 @@ public class SignalService { public void updateLastDownloadedReport(Integer id, String fileName) { mapper.setPackageLastDownloadedReport(id, fileName); } - - public List updateFlags(List subjects) { - List fidsPhoneDataList = phoneService.findFidsPhoneData(subjects, true); - LocalDateTime calculatedAt = LocalDateTime.now(); - fidsPhoneDataList.forEach(phoneData -> { - if (phoneData.getFid() != null) { - FlagsService.FlagsFidData flagsData = flagsService.findFlags(phoneData.getFid()); - mapper.registerSubjectsFidAndDebts(phoneData.getId(), phoneData.getFid(), flagsData.getFlagDebt5000(), flagsData.getFlag90days(), calculatedAt); - } else { - mapper.registerSubjectsFidAndDebts(phoneData.getId(), null, null, null, calculatedAt); - } - }); - - return fidsPhoneDataList; - } } diff --git a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java index b0d5c00..b094082 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java @@ -54,7 +54,7 @@ public class UpdatingFlagsService { private void updateFlags(Integer packageId, List subjects, String packageName) { try { signalService.setPackageStatus(packageId, Status.PROCESSING); - List phoneFidsData = signalService.updateFlags(subjects); + List phoneFidsData = signalService.updateFlags(subjects, packageId); if (!phoneFidsData.isEmpty()) { Signals signals = signalService.buildSignals(packageId, phoneFidsData); if (signalRestService.update(signals)) { diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 7a26b0d..f2bafa2 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -1,16 +1,15 @@ package ru.nbch.credit_tracker.service.indic; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.mapper.indic.FlagsMapper; -import java.math.BigDecimal; -import java.math.BigInteger; import java.time.LocalDate; -import java.util.Comparator; +import java.util.HashMap; import java.util.List; -import java.util.Optional; +import java.util.Map; import java.util.stream.Collectors; @Service @@ -22,11 +21,32 @@ public class FlagsService { this.mapper = mapper; } - public FlagsFidData findFlags(Integer fid) { - FlagsFidData flagsData = new FlagsFidData(); - flagsData.setFlagDebt5000(calcFlag5000(fid)); - flagsData.setFlag90days(calcFlag90Days(fid)); - return flagsData; + public Map findFlags(List fids) { + if (fids.isEmpty()) { + return new HashMap<>(); + } + Map result = new HashMap<>(); + Map flags5000 = calcFlag5000(fids); + Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(fids); + Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(fids); + + fids.forEach(fid -> { + Integer flagDebt5000 = flags5000.get(fid); + if (flagDebt5000 == null) { + flagDebt5000 = 0; + } + + Integer flag90days = flags90DaysByPastDueArrear.get(fid); + if (flag90days == null) { + flag90days = flags90DaysByRuPaymt.get(fid); + if (flag90days == null) { + flag90days = 0; + } + } + result.put(fid, new FlagsData(flagDebt5000, flag90days)); + }); + + return result; } /** @@ -34,27 +54,16 @@ public class FlagsService { * Если по хотя бы одной такой записи AMT_PAST_DUE > 5000, то флаг устанавливается в 1. * Если ни по одному счёту условие не выполняется или счетов вообще не найдено, флаг = 0. */ - private Integer calcFlag5000(Integer fid) { - List dataList = mapper.findFlagsDebt5000(fid).stream() - .collect(Collectors.groupingBy( - Flag5000FidData::getAccSerialNum, - Collectors.maxBy(Comparator.comparing(Flag5000FidData::getReportedDate)) - )) - .values() - .stream() - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); - - if (!dataList.isEmpty() && dataList.stream() - .anyMatch(item -> - item != null && item.getAmountPastDue() != null && - item.getAmountPastDue().compareTo(BigDecimal.valueOf(5000)) > 0) - ) { - return 1; - } else { - return 0; - } + private Map calcFlag5000(List fids) { + return mapper.findFlagsDebt5000(fids).stream() + .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag, + (existing, replacement) -> { + if (existing == 1) { + return existing; + } else { + return replacement; + } + })); } /** @@ -62,28 +71,44 @@ public class FlagsService { * В INDIC.RU_PASTDUEARREAR или RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. * Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0. */ - private Integer calcFlag90Days(Integer fid) { + private Map calcFlag90DaysByPastDueArrear(List fids) { LocalDate yearAgo = LocalDate.now().minusYears(1); - if (mapper.findFlag90ByPastDueArrear(fid, yearAgo)) { - return 1; - } else if (mapper.findFlag90ByRuPaymt(fid, yearAgo)) { - return 1; - } - return 0; + return mapper.findFlag90ByPastDueArrear(fids, yearAgo).stream() + .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag, + (existing, replacement) -> { + if (existing == 1) { + return existing; + } else { + return replacement; + } + })); + } + + private Map calcFlag90DaysByRuPaymt(List fids) { + LocalDate yearAgo = LocalDate.now().minusYears(1); + return mapper.findFlag90ByRuPaymt(fids, yearAgo).stream() + .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag, + (existing, replacement) -> { + if (existing == 1) { + return existing; + } else { + return replacement; + } + })); } @Getter @Setter - public static class FlagsFidData { + @AllArgsConstructor + public static class FlagsData { private Integer flagDebt5000; private Integer flag90days; } @Getter @Setter - public static class Flag5000FidData { - private BigInteger accSerialNum; - private LocalDate reportedDate; - private BigDecimal amountPastDue; + public static class FlagFidData { + private Integer fid; + private Integer flag; } } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java index b138846..0672f84 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java @@ -30,7 +30,8 @@ public class PhoneService { for (int i = 0; i < subjects.size(); i += batchSize) { int end = Math.min(i + batchSize, subjects.size()); List currentSubjects = subjects.subList(i, end); - List fidsData = mapper.findFids(currentSubjects.stream().map(subject -> "9" + subject.getPhone()).toList()); + List phones = currentSubjects.stream().map(subject -> "9" + subject.getPhone()).toList(); + List fidsData = mapper.findFids(phones); Map fidsMap = convertFidsDataToMap(fidsData); @@ -38,9 +39,9 @@ public class PhoneService { String phone = "9" + subject.getPhone(); Integer fid = fidsMap.get(phone); if (fid != null) { - fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), fid, phone, subject.getSubjectId())); + fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), fid, subject.getPhone(), subject.getSubjectId())); } else if (processNullFids) { - fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), null, phone, subject.getSubjectId())); + fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), null, subject.getPhone(), subject.getSubjectId())); } }); } diff --git a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml index 4764aa6..703c462 100644 --- a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml @@ -5,28 +5,73 @@ - - - - + + + - + SELECT rp.FID as fid, + CASE + WHEN EXISTS (SELECT 1 + FROM RU_PASTDUEARREAR rp2 + WHERE rp2.FID = rp.FID + AND rp2.ACC_SERIAL_NUM = rp.ACC_SERIAL_NUM + AND rp2.REPORTED_DT = rp.MAX_REP_DATE + AND rp2.AMT_PAST_DUE > 5000) THEN 1 + ELSE 0 + END as flag + FROM (SELECT FID, + ACC_SERIAL_NUM, + MAX(REPORTED_DT) as MAX_REP_DATE + FROM RU_PASTDUEARREAR + WHERE FID IN + + #{fid} + + GROUP BY FID, ACC_SERIAL_NUM) rp + GROUP BY rp.FID, rp.ACC_SERIAL_NUM, rp.MAX_REP_DATE - + SELECT rp.FID as fid, + CASE + WHEN EXISTS (SELECT 1 + FROM RU_PASTDUEARREAR rp2 + WHERE rp2.FID = rp.FID + AND rp2.REPORTED_DT >= #{reportedDate} + AND rp2.DAYS_PAST_DUE > 90) THEN 1 + ELSE 0 + END as flag + FROM (SELECT FID, + DAYS_PAST_DUE + FROM RU_PASTDUEARREAR + WHERE FID IN + + #{fid} + + GROUP BY FID, DAYS_PAST_DUE) rp + GROUP BY rp.FID, rp.DAYS_PAST_DUE - + SELECT rp.FID as fid, + CASE + WHEN EXISTS (SELECT 1 + FROM RU_PAYMT rp2 + WHERE rp2.FID = rp.FID + AND rp2.REPORTED_DT >= #{reportedDate} + AND rp2.DAYS_PAST_DUE > 90) THEN 1 + ELSE 0 + END as flag + FROM (SELECT FID, + DAYS_PAST_DUE + FROM RU_PAYMT + WHERE FID IN + + #{fid} + + GROUP BY FID, DAYS_PAST_DUE) rp + GROUP BY rp.FID, rp.DAYS_PAST_DUE \ No newline at end of file diff --git a/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml b/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml index 4c8083d..7dd9e60 100644 --- a/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml @@ -11,11 +11,11 @@ - SELECT NUMBER, FID, FILE_SINCE_DT FROM PHONE_NORM WHERE NUMBER in - + #{phone} ORDER BY FILE_SINCE_DT DESC, FID DESC diff --git a/src/main/resources/mybatis/mapper/signal/SignalMapper.xml b/src/main/resources/mybatis/mapper/signal/SignalMapper.xml index d4f614d..cfd25c2 100644 --- a/src/main/resources/mybatis/mapper/signal/SignalMapper.xml +++ b/src/main/resources/mybatis/mapper/signal/SignalMapper.xml @@ -6,50 +6,57 @@ - - - - - - - - + + + + + + + + - - - + + + - + UPDATE signals_user_packages SET status = #{status}, deactivated_at = #{deactivatedAt} - WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND package_name = #{packageName} AND status = #{previousStatus} + WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND package_name = #{packageName} AND status = + #{previousStatus} - INSERT INTO signals_user_packages (membercode, package_name, created_at, status) - values (#{request.userId}, #{request.packageName}, #{creadedAt}, #{status}) - RETURNING id + values (#{request.userId}, #{request.packageName}, #{creadedAt}, #{status}) RETURNING id - + INSERT INTO signals_user_packages (membercode, package_name, created_at, status) values (#{request.userId}, #{request.packageName}, #{creadedAt}, #{status}) @@ -70,6 +77,21 @@ + + + UPDATE signals_phone_fid_map + SET + package_id = #{subject.packageId}, + subject_id = #{subject.subjectId}, + phone = #{subject.phone}, + fid = #{subject.fid}, + flag_debt5000 = #{subject.flagDebt5000}, + flag_90_12 = #{subject.flag90days}, + calculated_at = #{subject.calculatedAt} + WHERE id = #{subject.id} + + + UPDATE signals_phone_fid_map SET fid = #{fid}, flag_debt5000 = #{flagDebt5000}, flag_90_12 = #{flag90}, calculated_at = #{calculatedAt} @@ -95,24 +117,37 @@ - SELECT membercode, package_name FROM signals_user_packages - WHERE id = #{id} LIMIT 1 + WHERE id = #{id} + LIMIT 1 - SELECT package_name, created_at FROM signals_user_packages - WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND status = #{status} + WHERE membercode IS NOT NULL + AND LEFT(membercode, 6) = #{userId} + AND status = #{status} - SELECT id, membercode, package_name, created_at, last_downloaded_report FROM signals_user_packages WHERE status = #{status}