treshenkov/credit_tracker
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use batchs for finding fids and flags

Browse source
This commit is contained in:
Mikhail Trofimov 2025-06-30 12:17:00 +03:00
parent 47450009ce
commit 4af0bad507
10 changed files with 244 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/main/java/ru/nbch/credit_tracker/entities/dto/SubjectData.java
View file

@ -1,12 +1,16 @@
package ru.nbch.credit_tracker.entities.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.time.LocalDateTime;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class SubjectData {
private Integer id;
private Integer packageId;

7
src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java
View file

@ -10,9 +10,10 @@ import java.util.List;
@Mapper
public interface FlagsMapper {
List<FlagsService.Flag5000FidData> findFlagsDebt5000(@Param("fid") Integer fid);
List<FlagsService.FlagFidData> findFlagsDebt5000(@Param("fids") List<Integer> fids);
boolean findFlag90ByPastDueArrear(@Param("fid") Integer fid, @Param("reportedDate") LocalDate reportedDate);
List<FlagsService.FlagFidData> findFlag90ByPastDueArrear(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
List<FlagsService.FlagFidData> findFlag90ByRuPaymt(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
boolean findFlag90ByRuPaymt(@Param("fid") Integer fid, @Param("reportedDate") LocalDate reportedDate);
}

2
src/main/java/ru/nbch/credit_tracker/mapper/signal/SignalMapper.java
View file

@ -42,5 +42,7 @@ public interface SignalMapper {
void setHits(@Param("hits") List<HitsData> hits);
void updateSubjects(@Param("subjects") List<SubjectData> subjects);
void setPackageLastDownloadedReport(@Param("id") Integer id, @Param("fileName") String fileName);
}

55
src/main/java/ru/nbch/credit_tracker/service/SignalService.java
View file

@ -1,5 +1,6 @@
package ru.nbch.credit_tracker.service;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
@ -26,8 +27,12 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Slf4j
@Service
public class SignalService {
@ -39,6 +44,8 @@ public class SignalService {
private final FlagsService flagsService;
private final PersonService personService;
private final int batchSize = 10000;
public SignalService(CreditTrackerSettings config,
SignalMapper mapper,
PhoneService phoneService,
@ -58,7 +65,6 @@ public class SignalService {
public Integer registerPackage(MonitoringRequest request) {
Integer packageId = mapper.registerPackage(request, LocalDateTime.now(), Status.PROCESSING.name());
int batchSize = 10000;
List<Subject> subjects = request.getSubjects().getSubject();
for (int i = 0; i < subjects.size(); i += batchSize) {
int end = Math.min(i + batchSize, subjects.size());
@ -69,7 +75,37 @@ public class SignalService {
public List<PhoneService.FidsPhoneData> registerSubjectsFidAndDebts(Integer packageId) {
List<SubjectData> subjects = mapper.findSubjectsData(packageId);
return updateFlags(subjects);
return updateFlags(subjects, packageId);
}
public List<PhoneService.FidsPhoneData> updateFlags(List<SubjectData> subjects, Integer packageId) {
List<PhoneService.FidsPhoneData> fidsPhoneDataList = phoneService.findFidsPhoneData(subjects, true);
LocalDateTime calculatedAt = LocalDateTime.now();
for (int i = 0; i < fidsPhoneDataList.size(); i += batchSize) {
int end = Math.min(i + batchSize, fidsPhoneDataList.size());
List<PhoneService.FidsPhoneData> currentList = fidsPhoneDataList.subList(i, end);
Map<Integer, FlagsService.FlagsData> flagsDataMap = flagsService.findFlags(currentList.stream().map(PhoneService.FidsPhoneData::getFid).filter(Objects::nonNull).toList());
List<SubjectData> subjectToUpdate = new ArrayList<>();
currentList.forEach(data -> {
SubjectData subject;
if (data.getFid() != null) {
FlagsService.FlagsData flagsData = flagsDataMap.get(data.getFid());
subject = new SubjectData(data.getId(), packageId, data.getSubjectId(), data.getPhone(), data.getFid(), flagsData.getFlagDebt5000(), flagsData.getFlag90days(), calculatedAt);
} else {
subject = new SubjectData(data.getId(), packageId, data.getSubjectId(), data.getPhone(),null, null, null, calculatedAt);
}
subjectToUpdate.add(subject);
});
int updateBatchSize = batchSize / 8; // 8 параметров на одну запись для обновления. Максимальное количество параметров ~65000
for (int j = 0; j < subjectToUpdate.size(); j += updateBatchSize) {
int endIndex = Math.min(j + updateBatchSize, subjectToUpdate.size());
mapper.updateSubjects(subjectToUpdate.subList(j, endIndex));
}
}
return fidsPhoneDataList;
}
public void rejectPackage(MonitoringRequest request) {
@ -196,19 +232,4 @@ public class SignalService {
public void updateLastDownloadedReport(Integer id, String fileName) {
mapper.setPackageLastDownloadedReport(id, fileName);
}
public List<PhoneService.FidsPhoneData> updateFlags(List<SubjectData> subjects) {
List<PhoneService.FidsPhoneData> fidsPhoneDataList = phoneService.findFidsPhoneData(subjects, true);
LocalDateTime calculatedAt = LocalDateTime.now();
fidsPhoneDataList.forEach(phoneData -> {
if (phoneData.getFid() != null) {
FlagsService.FlagsFidData flagsData = flagsService.findFlags(phoneData.getFid());
mapper.registerSubjectsFidAndDebts(phoneData.getId(), phoneData.getFid(), flagsData.getFlagDebt5000(), flagsData.getFlag90days(), calculatedAt);
} else {
mapper.registerSubjectsFidAndDebts(phoneData.getId(), null, null, null, calculatedAt);
}
});
return fidsPhoneDataList;
}
}

2
src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java
View file

@ -54,7 +54,7 @@ public class UpdatingFlagsService {
private void updateFlags(Integer packageId, List<SubjectData> subjects, String packageName) {
try {
signalService.setPackageStatus(packageId, Status.PROCESSING);
List<PhoneService.FidsPhoneData> phoneFidsData = signalService.updateFlags(subjects);
List<PhoneService.FidsPhoneData> phoneFidsData = signalService.updateFlags(subjects, packageId);
if (!phoneFidsData.isEmpty()) {
Signals signals = signalService.buildSignals(packageId, phoneFidsData);
if (signalRestService.update(signals)) {

103
src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java
View file

@ -1,16 +1,15 @@
package ru.nbch.credit_tracker.service.indic;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.mapper.indic.FlagsMapper;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.time.LocalDate;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@ -22,11 +21,32 @@ public class FlagsService {
this.mapper = mapper;
}
public FlagsFidData findFlags(Integer fid) {
FlagsFidData flagsData = new FlagsFidData();
flagsData.setFlagDebt5000(calcFlag5000(fid));
flagsData.setFlag90days(calcFlag90Days(fid));
return flagsData;
public Map<Integer, FlagsData> findFlags(List<Integer> fids) {
if (fids.isEmpty()) {
return new HashMap<>();
}
Map<Integer, FlagsData> result = new HashMap<>();
Map<Integer, Integer> flags5000 = calcFlag5000(fids);
Map<Integer, Integer> flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(fids);
Map<Integer, Integer> flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(fids);
fids.forEach(fid -> {
Integer flagDebt5000 = flags5000.get(fid);
if (flagDebt5000 == null) {
flagDebt5000 = 0;
}
Integer flag90days = flags90DaysByPastDueArrear.get(fid);
if (flag90days == null) {
flag90days = flags90DaysByRuPaymt.get(fid);
if (flag90days == null) {
flag90days = 0;
}
}
result.put(fid, new FlagsData(flagDebt5000, flag90days));
});
return result;
}
/**
@ -34,27 +54,16 @@ public class FlagsService {
* Если по хотя бы одной такой записи AMT_PAST_DUE > 5000, то флаг устанавливается в 1.
* Если ни по одному счёту условие не выполняется или счетов вообще не найдено, флаг = 0.
*/
private Integer calcFlag5000(Integer fid) {
List<Flag5000FidData> dataList = mapper.findFlagsDebt5000(fid).stream()
.collect(Collectors.groupingBy(
Flag5000FidData::getAccSerialNum,
Collectors.maxBy(Comparator.comparing(Flag5000FidData::getReportedDate))
))
.values()
.stream()
.filter(Optional::isPresent)
.map(Optional::get)
.toList();
if (!dataList.isEmpty() && dataList.stream()
.anyMatch(item ->
item != null && item.getAmountPastDue() != null &&
item.getAmountPastDue().compareTo(BigDecimal.valueOf(5000)) > 0)
) {
return 1;
private Map<Integer, Integer> calcFlag5000(List<Integer> fids) {
return mapper.findFlagsDebt5000(fids).stream()
.collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag,
(existing, replacement) -> {
if (existing == 1) {
return existing;
} else {
return 0;
return replacement;
}
}));
}
/**
@ -62,28 +71,44 @@ public class FlagsService {
* В INDIC.RU_PASTDUEARREAR или RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев.
* Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0.
*/
private Integer calcFlag90Days(Integer fid) {
private Map<Integer, Integer> calcFlag90DaysByPastDueArrear(List<Integer> fids) {
LocalDate yearAgo = LocalDate.now().minusYears(1);
if (mapper.findFlag90ByPastDueArrear(fid, yearAgo)) {
return 1;
} else if (mapper.findFlag90ByRuPaymt(fid, yearAgo)) {
return 1;
return mapper.findFlag90ByPastDueArrear(fids, yearAgo).stream()
.collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag,
(existing, replacement) -> {
if (existing == 1) {
return existing;
} else {
return replacement;
}
return 0;
}));
}
private Map<Integer, Integer> calcFlag90DaysByRuPaymt(List<Integer> fids) {
LocalDate yearAgo = LocalDate.now().minusYears(1);
return mapper.findFlag90ByRuPaymt(fids, yearAgo).stream()
.collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag,
(existing, replacement) -> {
if (existing == 1) {
return existing;
} else {
return replacement;
}
}));
}
@Getter
@Setter
public static class FlagsFidData {
@AllArgsConstructor
public static class FlagsData {
private Integer flagDebt5000;
private Integer flag90days;
}
@Getter
@Setter
public static class Flag5000FidData {
private BigInteger accSerialNum;
private LocalDate reportedDate;
private BigDecimal amountPastDue;
public static class FlagFidData {
private Integer fid;
private Integer flag;
}
}

7
src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java
View file

@ -30,7 +30,8 @@ public class PhoneService {
for (int i = 0; i < subjects.size(); i += batchSize) {
int end = Math.min(i + batchSize, subjects.size());
List<SubjectData> currentSubjects = subjects.subList(i, end);
List<FidsData> fidsData = mapper.findFids(currentSubjects.stream().map(subject -> "9" + subject.getPhone()).toList());
List<String> phones = currentSubjects.stream().map(subject -> "9" + subject.getPhone()).toList();
List<FidsData> fidsData = mapper.findFids(phones);
Map<String, Integer> fidsMap = convertFidsDataToMap(fidsData);
@ -38,9 +39,9 @@ public class PhoneService {
String phone = "9" + subject.getPhone();
Integer fid = fidsMap.get(phone);
if (fid != null) {
fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), fid, phone, subject.getSubjectId()));
fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), fid, subject.getPhone(), subject.getSubjectId()));
} else if (processNullFids) {
fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), null, phone, subject.getSubjectId()));
fidsPhoneDataList.add(new FidsPhoneData(subject.getId(), null, subject.getPhone(), subject.getSubjectId()));
}
});
}

73
src/main/resources/mybatis/mapper/indic/FlagsMapper.xml
View file

@ -5,28 +5,73 @@
<mapper namespace="ru.nbch.credit_tracker.mapper.indic.FlagsMapper">
<resultMap id="Flag5000FidDataResultMap" type="ru.nbch.credit_tracker.service.indic.FlagsService$Flag5000FidData">
<result property="accSerialNum" column="ACC_SERIAL_NUM" />
<result property="reportedDate" column="REPORTED_DT" />
<result property="amountPastDue" column="AMT_PAST_DUE" />
<resultMap id="FlagsFidDataResultMap" type="ru.nbch.credit_tracker.service.indic.FlagsService$FlagFidData">
<result property="fid" column="fid" />
<result property="flag" column="flag" />
</resultMap>
<select id="findFlagsDebt5000" resultMap="Flag5000FidDataResultMap">
SELECT ACC_SERIAL_NUM, AMT_PAST_DUE, REPORTED_DT
<select id="findFlagsDebt5000" resultMap="FlagsFidDataResultMap">
SELECT rp.FID as fid,
CASE
WHEN EXISTS (SELECT 1
FROM RU_PASTDUEARREAR rp2
WHERE rp2.FID = rp.FID
AND rp2.ACC_SERIAL_NUM = rp.ACC_SERIAL_NUM
AND rp2.REPORTED_DT = rp.MAX_REP_DATE
AND rp2.AMT_PAST_DUE > 5000) THEN 1
ELSE 0
END as flag
FROM (SELECT FID,
ACC_SERIAL_NUM,
MAX(REPORTED_DT) as MAX_REP_DATE
FROM RU_PASTDUEARREAR
WHERE FID = #{fid}
ORDER BY REPORTED_DT DESC
WHERE FID IN
<foreach item="fid" collection="fids" open="(" separator="," close=")">
#{fid}
</foreach>
GROUP BY FID, ACC_SERIAL_NUM) rp
GROUP BY rp.FID, rp.ACC_SERIAL_NUM, rp.MAX_REP_DATE
</select>
<select id="findFlag90ByPastDueArrear">
SELECT COUNT(*) > 0
<select id="findFlag90ByPastDueArrear" resultMap="FlagsFidDataResultMap">
SELECT rp.FID as fid,
CASE
WHEN EXISTS (SELECT 1
FROM RU_PASTDUEARREAR rp2
WHERE rp2.FID = rp.FID
AND rp2.REPORTED_DT >= #{reportedDate}
AND rp2.DAYS_PAST_DUE > 90) THEN 1
ELSE 0
END as flag
FROM (SELECT FID,
DAYS_PAST_DUE
FROM RU_PASTDUEARREAR
WHERE FID = #{fid} AND DAYS_PAST_DUE >= 90 AND REPORTED_DT >= #{reportedDate}
WHERE FID IN
<foreach item="fid" collection="fids" open="(" separator="," close=")">
#{fid}
</foreach>
GROUP BY FID, DAYS_PAST_DUE) rp
GROUP BY rp.FID, rp.DAYS_PAST_DUE
</select>
<select id="findFlag90ByRuPaymt">
SELECT COUNT(*) > 0
<select id="findFlag90ByRuPaymt" resultMap="FlagsFidDataResultMap">
SELECT rp.FID as fid,
CASE
WHEN EXISTS (SELECT 1
FROM RU_PAYMT rp2
WHERE rp2.FID = rp.FID
AND rp2.REPORTED_DT >= #{reportedDate}
AND rp2.DAYS_PAST_DUE > 90) THEN 1
ELSE 0
END as flag
FROM (SELECT FID,
DAYS_PAST_DUE
FROM RU_PAYMT
WHERE FID = #{fid} AND DAYS_PAST_DUE >= 90 AND REPORTED_DT >= #{reportedDate}
WHERE FID IN
<foreach item="fid" collection="fids" open="(" separator="," close=")">
#{fid}
</foreach>
GROUP BY FID, DAYS_PAST_DUE) rp
GROUP BY rp.FID, rp.DAYS_PAST_DUE
</select>
</mapper>

4
src/main/resources/mybatis/mapper/indic/PhoneMapper.xml
View file

@ -11,11 +11,11 @@
<result property="fileSinceDate" column="FILE_SINCE_DT" />
</resultMap>
<select id="findFids" resultMap="fidsDataResultMap" parameterType="java.util.List" resultType="ru.nbch.credit_tracker.service.indic.PhoneService$FidsData">
<select id="findFids" resultMap="fidsDataResultMap">
SELECT NUMBER, FID, FILE_SINCE_DT
FROM PHONE_NORM
WHERE NUMBER in
<foreach item='phone' collection='phones' open='(' separator=',' close=')'>
<foreach item="phone" collection="phones" open="(" separator="," close=")">
#{phone}
</foreach>
ORDER BY FILE_SINCE_DT DESC, FID DESC

61
src/main/resources/mybatis/mapper/signal/SignalMapper.xml
View file

@ -28,28 +28,35 @@
<select id="isPackageNotUnique" resultType="boolean">
SELECT COUNT(*) > 0
FROM signals_user_packages
WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND package_name = #{packageName}
WHERE membercode IS NOT NULL
AND LEFT(membercode, 6) = #{userId}
AND package_name = #{packageName}
</select>
<select id="isPackageExists" resultType="boolean">
SELECT COUNT(*) > 0
FROM signals_user_packages
WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND package_name = #{packageName} AND status = #{status}
WHERE membercode IS NOT NULL
AND LEFT(membercode, 6) = #{userId}
AND package_name = #{packageName}
AND status = #{status}
</select>
<update id="deletePackage">
UPDATE signals_user_packages
SET status = #{status}, deactivated_at = #{deactivatedAt}
WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND package_name = #{packageName} AND status = #{previousStatus}
WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND package_name = #{packageName} AND status =
#{previousStatus}
</update>
<select id="registerPackage" resultType="java.lang.Integer" parameterType="ru.nbch.credit_tracker.entities.request.monitoring_request.MonitoringRequest" >
<select id="registerPackage" resultType="java.lang.Integer"
parameterType="ru.nbch.credit_tracker.entities.request.monitoring_request.MonitoringRequest">
INSERT INTO signals_user_packages (membercode, package_name, created_at, status)
values (#{request.userId}, #{request.packageName}, #{creadedAt}, #{status})
RETURNING id
values (#{request.userId}, #{request.packageName}, #{creadedAt}, #{status}) RETURNING id
</select>
<insert id="insertPackage" parameterType="ru.nbch.credit_tracker.entities.request.monitoring_request.MonitoringRequest" >
<insert id="insertPackage"
parameterType="ru.nbch.credit_tracker.entities.request.monitoring_request.MonitoringRequest">
INSERT INTO signals_user_packages (membercode, package_name, created_at, status)
values (#{request.userId}, #{request.packageName}, #{creadedAt}, #{status})
</insert>
@ -70,6 +77,21 @@
</foreach>
</insert>
<update id="updateSubjects" parameterType="java.util.List">
<foreach collection="subjects" item="subject" separator=";">
UPDATE signals_phone_fid_map
SET
package_id = #{subject.packageId},
subject_id = #{subject.subjectId},
phone = #{subject.phone},
fid = #{subject.fid},
flag_debt5000 = #{subject.flagDebt5000},
flag_90_12 = #{subject.flag90days},
calculated_at = #{subject.calculatedAt}
WHERE id = #{subject.id}
</foreach>
</update>
<update id="registerSubjectsFidAndDebts">
UPDATE signals_phone_fid_map
SET fid = #{fid}, flag_debt5000 = #{flagDebt5000}, flag_90_12 = #{flag90}, calculated_at = #{calculatedAt}
@ -95,24 +117,37 @@
</select>
<select id="findFullSubjectsData" resultMap="subjectDataResultMap" resultType="java.util.List">
SELECT id, package_id, subject_id, phone, fid, flag_90_12, flag_debt5000, calculated_at
SELECT id,
package_id,
subject_id,
phone,
fid,
flag_90_12,
flag_debt5000,
calculated_at
FROM signals_phone_fid_map
WHERE package_id = #{packageId}
</select>
<select id="findPackageData" resultMap="packageDataResultMap" resultType="ru.nbch.credit_tracker.entities.dto.PackageData">
<select id="findPackageData" resultMap="packageDataResultMap"
resultType="ru.nbch.credit_tracker.entities.dto.PackageData">
SELECT membercode, package_name
FROM signals_user_packages
WHERE id = #{id} LIMIT 1
WHERE id = #{id}
LIMIT 1
</select>
<select id="findPackageDataList" resultMap="packageDataResultMap" resultType="ru.nbch.credit_tracker.entities.dto.PackageData">
<select id="findPackageDataList" resultMap="packageDataResultMap"
resultType="ru.nbch.credit_tracker.entities.dto.PackageData">
SELECT package_name, created_at
FROM signals_user_packages
WHERE membercode IS NOT NULL AND LEFT(membercode, 6) = #{userId} AND status = #{status}
WHERE membercode IS NOT NULL
AND LEFT(membercode, 6) = #{userId}
AND status = #{status}
</select>
<select id="findPackagesByStatus" resultMap="packageDataResultMap" resultType="ru.nbch.credit_tracker.entities.dto.PackageData">
<select id="findPackagesByStatus" resultMap="packageDataResultMap"
resultType="ru.nbch.credit_tracker.entities.dto.PackageData">
SELECT id, membercode, package_name, created_at, last_downloaded_report
FROM signals_user_packages
WHERE status = #{status}