From bddc5bb1fa7a097f7f00b6db8b67fbd45a5b6507 Mon Sep 17 00:00:00 2001 From: etreschenkov Date: Mon, 22 Sep 2025 22:02:55 +0300 Subject: [PATCH 01/20] logging rest request; logging query time for every mapper; optimization query and refactoring some methods --- pom.xml | 139 ++++++++++-------- .../component/MapperTimingAspect.java | 28 ++++ .../rest/filters/HttpLoggingFilter.java | 111 ++++++++++++++ .../mapper/signal/PhoneFidMapMapper.java | 1 + .../service/MonitoringService.java | 37 +++-- .../credit_tracker/service/SignalService.java | 26 +++- .../service/UpdatingFlagsService.java | 17 +-- .../service/indic/FlagsService.java | 77 +++++----- .../service/indic/PhoneService.java | 47 +++++- src/main/resources/logback.xml | 2 +- .../mybatis/mapper/indic/FlagsMapper.xml | 106 ++++++------- .../mybatis/mapper/indic/PhoneMapper.xml | 15 +- .../mapper/signal/PhoneFidMapMapper.xml | 11 ++ .../service/PhoneServiceTest.java | 57 +++++++ 14 files changed, 482 insertions(+), 192 deletions(-) create mode 100644 src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java create mode 100644 src/main/java/ru/nbch/credit_tracker/config/rest/filters/HttpLoggingFilter.java create mode 100644 src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java diff --git a/pom.xml b/pom.xml index f876eb6..ffd17aa 100644 --- a/pom.xml +++ b/pom.xml @@ -27,6 +27,10 @@ org.springframework.boot spring-boot-starter-web + + org.springframework.boot + spring-boot-starter-aop + org.mybatis.spring.boot mybatis-spring-boot-starter @@ -122,8 +126,83 @@ 3.0.4 test + + org.mockito + mockito-junit-jupiter + test + + + + docker-push-dev + + + + + io.fabric8 + docker-maven-plugin + 0.45.0 + + + Build and deploy docker container + install + + + push + + + + + + + ${project.artifactId}:${project.version} + ${registry.url} + + ${project.basedir}/Dockerfile + + + + + ${project.build.directory}/${project.build.finalName}.jar + . + + + ${project.basedir}/apm-agent/elastic-apm-agent-1.45.0.jar + . + + + ${project.basedir}/start-credit-tracker.sh + . + + + ${project.basedir}/credit-tracker.properties + . + + + ${project.basedir}/monitoring-request.xsd + . + + + ${project.basedir}/signal-package.xsd + . + + + + + + ${project.version} + + + + + + + + + + + credit-tracker @@ -181,65 +260,7 @@ - - io.fabric8 - docker-maven-plugin - 0.45.0 - - - Build and deploy docker container - install - - build - push - - - - - - - ${project.artifactId}:${project.version} - ${registry.url} - - ${project.basedir}/Dockerfile - - - - - ${project.build.directory}/${project.build.finalName}.jar - . - - - ${project.basedir}/apm-agent/elastic-apm-agent-1.45.0.jar - . - - - ${project.basedir}/start-credit-tracker.sh - . - - - ${project.basedir}/credit-tracker.properties - . - - - ${project.basedir}/monitoring-request.xsd - . - - - ${project.basedir}/signal-package.xsd - . - - - - - - ${project.version} - - - - - - + diff --git a/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java b/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java new file mode 100644 index 0000000..7802484 --- /dev/null +++ b/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java @@ -0,0 +1,28 @@ +package ru.nbch.credit_tracker.component; + +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.stereotype.Component; + +@Aspect +@Slf4j +@Component +public class MapperTimingAspect { + @Around("within(@org.apache.ibatis.annotations.Mapper *)") + public Object timeMapper(ProceedingJoinPoint pjp) throws Throwable { + long start = System.nanoTime(); + try { + return pjp.proceed(); + } finally { + long tookMs = (System.nanoTime() - start) / 1_000_000; + String method = pjp.getSignature().toShortString(); + if (tookMs > 200) { + log.warn("Slow mapper call {} took {} ms", method, tookMs); + } else { + log.debug("Mapper call {} took {} ms", method, tookMs); + } + } + } +} diff --git a/src/main/java/ru/nbch/credit_tracker/config/rest/filters/HttpLoggingFilter.java b/src/main/java/ru/nbch/credit_tracker/config/rest/filters/HttpLoggingFilter.java new file mode 100644 index 0000000..bd8bd3c --- /dev/null +++ b/src/main/java/ru/nbch/credit_tracker/config/rest/filters/HttpLoggingFilter.java @@ -0,0 +1,111 @@ +package ru.nbch.credit_tracker.config.rest.filters; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.util.ContentCachingRequestWrapper; +import org.springframework.web.util.ContentCachingResponseWrapper; + +@Component +@Slf4j +@Order(0) +public class HttpLoggingFilter extends OncePerRequestFilter { + + private static final Set SENSITIVE_HEADERS = Set.of("authorization", "cookie", "set-cookie"); + private static final int MAX_BODY = 10 * 1024; + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + String path = request.getRequestURI(); + return path.startsWith("/actuator/health") || path.startsWith("/swagger") + || path.startsWith("/v3/api-docs") || path.startsWith("/static/"); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + + long startNs = System.nanoTime(); + + ContentCachingRequestWrapper req = wrapRequest(request); + ContentCachingResponseWrapper resp = new ContentCachingResponseWrapper(response); + + String correlationId = Optional.ofNullable(request.getHeader("X-Correlation-Id")) + .filter(StringUtils::hasText) + .orElse(UUID.randomUUID().toString()); + resp.setHeader("X-Correlation-Id", correlationId); + + try { + chain.doFilter(req, resp); + } finally { + long tookMs = (System.nanoTime() - startNs) / 1_000_000; + + String requestBody = readBody(req.getContentAsByteArray(), req.getCharacterEncoding()); + String responseBody = readBody(resp.getContentAsByteArray(), resp.getCharacterEncoding()); + + Map reqHeaders = headersMap(req); + Map respHeaders = headersMap(resp); + + mask(reqHeaders); + mask(respHeaders); + + log.info( + "HTTP {} {}{} | status={} | took={}ms | cid={} | reqHeaders={} | reqBody={} | respHeaders={} | respBody={}", + req.getMethod(), + req.getRequestURI(), + req.getQueryString() == null ? "" : "?" + req.getQueryString(), + resp.getStatus(), + tookMs, + correlationId, + reqHeaders, + requestBody, + respHeaders, + responseBody + ); + + resp.copyBodyToResponse(); + } + } + + private static ContentCachingRequestWrapper wrapRequest(HttpServletRequest request) { + return (request instanceof ContentCachingRequestWrapper c) ? c : new ContentCachingRequestWrapper(request, MAX_BODY); + } + + private static String readBody(byte[] buf, String encoding) { + if (buf == null || buf.length == 0) return ""; + var bytes = buf.length > MAX_BODY ? Arrays.copyOf(buf, MAX_BODY) : buf; + String body = new String(bytes, encoding != null ? java.nio.charset.Charset.forName(encoding) : StandardCharsets.UTF_8); + if (buf.length > MAX_BODY) body += "…[truncated]"; + return body; + } + + private static Map headersMap(HttpServletRequest req) { + return Collections.list(req.getHeaderNames()).stream() + .collect(Collectors.toMap(h -> h, req::getHeader, (a, b) -> a, LinkedHashMap::new)); + } + + private static Map headersMap(HttpServletResponse resp) { + return resp.getHeaderNames().stream() + .collect(Collectors.toMap(h -> h, resp::getHeader, (a, b) -> a, LinkedHashMap::new)); + } + + private static void mask(Map headers) { + headers.replaceAll((k, v) -> SENSITIVE_HEADERS.contains(k.toLowerCase()) ? "***" : v); + } +} diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java index 0e42995..80f216f 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java @@ -15,6 +15,7 @@ public interface PhoneFidMapMapper { List findFullPhoneFidMap(@Param("packageId") Integer packageId, @Param("offset") int offset, @Param("limit") int limit); void updatePhoneFidMaps(@Param("subjects") List subjects); + void updatePhoneFid(@Param("subject") PhoneFidMap subject); void updatePhoneFidMapsErrors(@Param("subjects") List subjects); diff --git a/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java b/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java index 1f30550..1a8e25f 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java @@ -1,8 +1,22 @@ package ru.nbch.credit_tracker.service; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; import ru.nbch.credit_tracker.entities.dto.Hit; @@ -22,21 +36,6 @@ import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlWr import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor; import ru.nbch.credit_tracker.utils.CTUtil; -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - /** * Сервис периодического опроса официального сервиса сигналов на наличие новых отчетов по пакетам находящимся в статусе ON_MONITORING */ @@ -60,9 +59,9 @@ public class MonitoringService { } public void run() { - ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-")); - long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15; - scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES); +// ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-")); +// long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15; +// scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES); } private void monitor() { diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index 138a075..eb7db82 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -15,6 +15,7 @@ import ru.nbch.credit_tracker.entities.external_request.signals.Signals; import ru.nbch.credit_tracker.entities.external_response.monitoring_report_error.MonitoringReportErrors; import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest; import ru.nbch.credit_tracker.enums.Status; +import ru.nbch.credit_tracker.mapper.signal.PhoneFidMapMapper; import ru.nbch.credit_tracker.service.indic.FlagsService; import ru.nbch.credit_tracker.service.indic.PersonService; import ru.nbch.credit_tracker.service.indic.PhoneService; @@ -43,6 +44,7 @@ public class SignalService { private final HitsService hitsService; private final UserPackageService userPackageService; private final PhoneFidMapService phoneFidMapService; + private final PhoneFidMapMapper phoneFidMapMapper; private final PhoneService phoneService; private final FlagsService flagsService; private final PersonService personService; @@ -54,6 +56,7 @@ public class SignalService { HitsService hitsService, UserPackageService userPackageService, PhoneFidMapService phoneFidMapService, + PhoneFidMapMapper mapper, PhoneService phoneService, FlagsService flagsService, PersonService personService, @@ -62,6 +65,7 @@ public class SignalService { this.hitsService = hitsService; this.userPackageService = userPackageService; this.phoneFidMapService = phoneFidMapService; + this.phoneFidMapMapper = mapper; this.phoneService = phoneService; this.flagsService = flagsService; this.personService = personService; @@ -125,7 +129,7 @@ public class SignalService { } if (!filteredItems.isEmpty()) { - boolean fidsFound = updateFlags(filteredItems, packageId); + boolean fidsFound = updateFlags2(filteredItems, packageId); if (noFidsFound) { noFidsFound = !fidsFound; } @@ -187,6 +191,26 @@ public class SignalService { return fidsFound; } + public boolean updateFlags2(List subjects, Integer packageId) { + log.info("Start calculating fids and flags for subjects batch. PackageId: {}", packageId); + Map fidsMap = phoneService.defineFidByPhone(subjects); + Map flagsDataMap = flagsService.findFlags(fidsMap.keySet().stream().toList()); + for (Map.Entry entry : flagsDataMap.entrySet()) { + Integer fid = entry.getKey(); + FlagsService.FlagsData flagsData = entry.getValue(); + PhoneFidMap phoneFidMap = fidsMap.get(fid); + + phoneFidMap.setFid(fid); + phoneFidMap.setFlagDebt5000(flagsData.getFlagDebt5000()); + phoneFidMap.setFlag90days(flagsData.getFlag90days()); + phoneFidMap.setCalculatedAt(LocalDateTime.now()); + + phoneFidMapMapper.updatePhoneFid(phoneFidMap); + } + log.info("Finish calculating fids and flags for subjects. PackageId: {}. Batch size: {}", packageId, subjects.size()); + return !fidsMap.isEmpty(); + } + public void rejectPackage(MonitoringRequest request) { UserPackage userPackage = UserPackage.builder() .userId(request.getUserId()) diff --git a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java index 19549fb..6b0a373 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java @@ -1,7 +1,10 @@ package ru.nbch.credit_tracker.service; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.function.Predicate; import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; @@ -9,14 +12,6 @@ import ru.nbch.credit_tracker.entities.dto.UserPackage; import ru.nbch.credit_tracker.enums.Status; import ru.nbch.credit_tracker.service.signal.SignalRestService; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; - /** * Сервис периодического обновления фидов и флагов просрочек по пакетам находящимся в статусе ON_MONITORING * По обновленным пакетам формируется новый запрос постановки на мониторинг в официальный сервис сигналов @@ -43,8 +38,8 @@ public class UpdatingFlagsService { } public void run() { - ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("updating-flags-")); - scheduler.scheduleAtFixedRate(this::update, 0, updateFlagsInterval, TimeUnit.DAYS); +// ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("updating-flags-")); +// scheduler.scheduleAtFixedRate(this::update, 0, updateFlagsInterval, TimeUnit.DAYS); } private void update() { diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index f2bafa2..5a11f68 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -1,20 +1,21 @@ package ru.nbch.credit_tracker.service.indic; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import org.springframework.stereotype.Service; -import ru.nbch.credit_tracker.mapper.indic.FlagsMapper; - import java.time.LocalDate; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import ru.nbch.credit_tracker.mapper.indic.FlagsMapper; @Service public class FlagsService { + private static final Logger log = LoggerFactory.getLogger(FlagsService.class); private final FlagsMapper mapper; public FlagsService(FlagsMapper mapper) { @@ -25,11 +26,13 @@ public class FlagsService { if (fids.isEmpty()) { return new HashMap<>(); } - Map result = new HashMap<>(); - Map flags5000 = calcFlag5000(fids); - Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(fids); - Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(fids); + List distinctFids = fids.stream().distinct().toList(); + log.debug("Fids count: {}", distinctFids.size()); + Map flags5000 = calcFlag5000(distinctFids); + Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(distinctFids); + Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); + Map result = new HashMap<>(); fids.forEach(fid -> { Integer flagDebt5000 = flags5000.get(fid); if (flagDebt5000 == null) { @@ -55,15 +58,11 @@ public class FlagsService { * Если ни по одному счёту условие не выполняется или счетов вообще не найдено, флаг = 0. */ private Map calcFlag5000(List fids) { - return mapper.findFlagsDebt5000(fids).stream() - .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag, - (existing, replacement) -> { - if (existing == 1) { - return existing; - } else { - return replacement; - } - })); + log.debug("Start calculating flag 5000"); + List rows = mapper.findFlagsDebt5000(fids); + Map res = makeFidFlagMapByRs(fids, rows); + log.debug("Complete calculating flag 5000"); + return res; } /** @@ -72,31 +71,33 @@ public class FlagsService { * Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0. */ private Map calcFlag90DaysByPastDueArrear(List fids) { + log.debug("Start calculating flag 90 by PastDueArrear"); LocalDate yearAgo = LocalDate.now().minusYears(1); - return mapper.findFlag90ByPastDueArrear(fids, yearAgo).stream() - .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag, - (existing, replacement) -> { - if (existing == 1) { - return existing; - } else { - return replacement; - } - })); + List rows = mapper.findFlag90ByPastDueArrear(fids, yearAgo); + Map res = makeFidFlagMapByRs(fids, rows); + log.debug("Complete calculating flag 90 by PastDueArrear"); + return res; } private Map calcFlag90DaysByRuPaymt(List fids) { + log.debug("Start calculating flag 90 by ruPaymt"); LocalDate yearAgo = LocalDate.now().minusYears(1); - return mapper.findFlag90ByRuPaymt(fids, yearAgo).stream() - .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag, - (existing, replacement) -> { - if (existing == 1) { - return existing; - } else { - return replacement; - } - })); + List rows = mapper.findFlag90ByRuPaymt(fids, yearAgo); + Map res = makeFidFlagMapByRs(fids, rows); + log.debug("Complete calculating flag 90 by ruPaymt"); + return res; } + private Map makeFidFlagMapByRs(List fids, List rows) { + Map result = new HashMap<>(); + for (Integer fid : fids) { + result.put(fid, 0); + } + for (FlagFidData row : rows) { + result.put(row.getFid(), row.getFlag() != null ? row.getFlag() : 0); + } + return result; + } @Getter @Setter @AllArgsConstructor diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java index d063be1..5b8e1a6 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java @@ -1,17 +1,19 @@ package ru.nbch.credit_tracker.service.indic; -import lombok.Getter; -import lombok.Setter; -import org.springframework.stereotype.Service; -import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; -import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; - import java.time.LocalDate; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.springframework.stereotype.Service; +import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; +import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; @Service public class PhoneService { @@ -31,6 +33,37 @@ public class PhoneService { return convertFidsDataToMap(mapper.findFids(phones)); } + public Map defineFidByPhone(List subjects) { + Map phoneFidMap = new HashMap<>(); + List phones = subjects.stream(). + map(subject -> "9" + subject.getPhone()) + .toList(); + + subjects.sort( + Comparator.comparing( + PhoneFidMap::getPhone, + Comparator.naturalOrder() + ) + ); + + //отсортированный список по NUMBER, FID и FILE_SINCE_DT + List phonesByFid = mapper.findFids(phones); + Iterator iterator = phonesByFid.iterator(); + for (PhoneFidMap phoneFid : subjects) { + while (iterator.hasNext()) { + FidsData fidsData = iterator.next(); + if (fidsData.getPhone().equals(phoneFid.getPhone())) { + //fixme доступ по ссылке, страшно, придумать что-то еще + phoneFidMap.put(fidsData.getFid(), phoneFid); + iterator.remove(); + break; + } + iterator.remove(); + } + } + return phoneFidMap; + } + private Map convertFidsDataToMap(List fidsData) { if (fidsData == null || fidsData.isEmpty()) { return new HashMap<>(); @@ -50,6 +83,8 @@ public class PhoneService { @Getter @Setter + @AllArgsConstructor + @NoArgsConstructor public static class FidsData { private Integer fid; private String phone; diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 0825d76..88ce91c 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -8,7 +8,7 @@ - %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n utf-8 diff --git a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml index 24572db..ccdde14 100644 --- a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml @@ -11,70 +11,70 @@ \ No newline at end of file diff --git a/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml b/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml index bd754a9..856a06d 100644 --- a/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/PhoneMapper.xml @@ -12,14 +12,21 @@ \ No newline at end of file diff --git a/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml b/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml index 9d668d9..3ff64c4 100644 --- a/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml +++ b/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml @@ -42,6 +42,17 @@ + + UPDATE signals_phone_fid_map + SET + fid = #{subject.fid}, + flag_debt5000 = #{subject.flagDebt5000}, + flag_90_12 = #{subject.flag90days}, + calculated_at = #{subject.calculatedAt}, + error_text = #{subject.errorText} + WHERE id = #{subject.id} + + UPDATE signals_phone_fid_map diff --git a/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java b/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java new file mode 100644 index 0000000..cba98b0 --- /dev/null +++ b/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java @@ -0,0 +1,57 @@ +package ru.nbch.credit_tracker.service; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import static org.mockito.Mockito.when; +import org.mockito.junit.jupiter.MockitoExtension; +import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; +import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; +import ru.nbch.credit_tracker.service.indic.PhoneService; + +@ExtendWith(MockitoExtension.class) +public class PhoneServiceTest { + + @Mock + PhoneMapper phoneMapper; + + @Test + public void defineFidByPhoneTest() { + PhoneService phoneService = new PhoneService(phoneMapper); + + fidsDataWithEqualSinceDt.sort(byDbOrder); + when(phoneMapper.findFids(Mockito.anyList())).thenReturn(fidsDataWithEqualSinceDt); + + List phoneFidMap = new ArrayList<>(); + phoneFidMap.add(PhoneFidMap.builder().phone("1111").build()); + phoneFidMap.add(PhoneFidMap.builder().phone("1112").build()); + phoneFidMap.add(PhoneFidMap.builder().phone("1114").build()); + + Map res = phoneService.defineFidByPhone(phoneFidMap); + Assertions.assertEquals(3, res.size()); + Assertions.assertEquals("1111", res.get(124).getPhone()); + Assertions.assertEquals("1112", res.get(125).getPhone()); + Assertions.assertEquals("1114", res.get(126).getPhone()); + } + + private final List fidsDataWithEqualSinceDt = new ArrayList<>( + List.of( + new PhoneService.FidsData(123, "1111", LocalDate.of(2025, 9, 10)), + new PhoneService.FidsData(124, "1111", LocalDate.of(2025, 9, 10)), + new PhoneService.FidsData(125, "1112", LocalDate.of(2024, 10, 10)), + new PhoneService.FidsData(126, "1114", LocalDate.of(2024, 10, 11)) + ) + ); + + Comparator byDbOrder = + Comparator.comparing(PhoneService.FidsData::getPhone, Comparator.nullsFirst(Comparator.naturalOrder())) // pn.number ASC + .thenComparing(PhoneService.FidsData::getFileSinceDate, Comparator.nullsLast(Comparator.reverseOrder())) // file_since_dt DESC + .thenComparing(PhoneService.FidsData::getFid, Comparator.reverseOrder()); +} From 29b017c51c4d987c1f8839b121b2dce394a29206 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Tue, 23 Sep 2025 13:43:57 +0300 Subject: [PATCH 02/20] Minor fix --- .../component/MapperTimingAspect.java | 2 +- .../credit_tracker/service/SignalService.java | 49 +------------------ .../UpdateProcessingPackagesService.java | 17 +++---- .../service/indic/PhoneService.java | 13 ++--- .../mybatis/mapper/indic/FlagsMapper.xml | 1 + 5 files changed, 16 insertions(+), 66 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java b/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java index 7802484..e0f818a 100644 --- a/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java +++ b/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java @@ -18,7 +18,7 @@ public class MapperTimingAspect { } finally { long tookMs = (System.nanoTime() - start) / 1_000_000; String method = pjp.getSignature().toShortString(); - if (tookMs > 200) { + if (tookMs > 500) { log.warn("Slow mapper call {} took {} ms", method, tookMs); } else { log.debug("Mapper call {} took {} ms", method, tookMs); diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index eb7db82..d5c80dc 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -129,7 +129,7 @@ public class SignalService { } if (!filteredItems.isEmpty()) { - boolean fidsFound = updateFlags2(filteredItems, packageId); + boolean fidsFound = updateFlags(filteredItems, packageId); if (noFidsFound) { noFidsFound = !fidsFound; } @@ -145,53 +145,8 @@ public class SignalService { * Обрабатывает пачку PhoneFidMap: обновляет фиды и флаги просрочек * @return fidsFound true - если хотя бы по одному номеру был найден фид, false - в противном случае */ + // TODO Добавить отдельный метод. Если не нашелся фид, надо обновить в базе fid = null и флаги = null calculated_at=now() public boolean updateFlags(List subjects, Integer packageId) { - log.info("Start calculating fids and flags for subjects batch. PackageId: {}", packageId); - Map fidsMap = phoneService.findFidsPhoneData(subjects); - LocalDateTime calculatedAt = LocalDateTime.now(); - boolean fidsFound = fidsMap.values().stream().anyMatch(Objects::nonNull); - Map flagsDataMap = flagsService.findFlags(fidsMap.values().stream().filter(Objects::nonNull).toList()); - List subjectToUpdate = new ArrayList<>(); - - subjects.forEach(subject -> { - Integer fid = fidsMap.get("9" + subject.getPhone()); - if (fid != null) { - FlagsService.FlagsData flagsData = flagsDataMap.get(fid); - subject = PhoneFidMap.builder() - .id(subject.getId()) - .packageId(packageId) - .subjectId(subject.getSubjectId()) - .phone(subject.getPhone()) - .fid(fid) - .flagDebt5000(flagsData.getFlagDebt5000()) - .flag90days(flagsData.getFlag90days()) - .calculatedAt(calculatedAt) - .build(); - } else { - subject = PhoneFidMap.builder() - .id(subject.getId()) - .packageId(packageId) - .subjectId(subject.getSubjectId()) - .phone(subject.getPhone()) - .calculatedAt(calculatedAt) - .build(); - } - subjectToUpdate.add(subject); - }); - - int updateBatchSize = batchSize / 9; // 9 параметров на одну запись для обновления. Максимальное количество параметров ~65000 - for (int j = 0; j < subjectToUpdate.size(); j += updateBatchSize) { - int endIndex = Math.min(j + updateBatchSize, subjectToUpdate.size()); - phoneFidMapService.updatePhoneFidMaps(subjectToUpdate.subList(j, endIndex)); - log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjectToUpdate.size(), packageId); - } - - log.info("Finish calculating fids and flags for subjects. PackageId: {}. Batch size: {}", packageId, subjects.size()); - - return fidsFound; - } - - public boolean updateFlags2(List subjects, Integer packageId) { log.info("Start calculating fids and flags for subjects batch. PackageId: {}", packageId); Map fidsMap = phoneService.defineFidByPhone(subjects); Map flagsDataMap = flagsService.findFlags(fidsMap.keySet().stream().toList()); diff --git a/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java b/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java index 2888c82..5630238 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java @@ -2,12 +2,9 @@ package ru.nbch.credit_tracker.service; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import ru.nbch.credit_tracker.entities.dto.UserPackage; import ru.nbch.credit_tracker.enums.Status; import ru.nbch.credit_tracker.service.signal.SignalRestService; -import java.util.List; - /** * Сервис для завершения обработки пакетов находящихся в статусе PROCESSING * Запускается при старте приложения и отрабатывает 1 раз @@ -29,13 +26,13 @@ public class UpdateProcessingPackagesService { } public void run() { - log.info("Start updating processing packages"); - List packages = signalService.findPackagesOnProcessing(); - for (UserPackage userPackage : packages) { - log.info("Updating flags for Package {}", userPackage.getPackageName()); - process(userPackage.getId(), userPackage.getPackageName()); - } - log.info("Finish updating processing packages"); +// log.info("Start updating processing packages"); +// List packages = signalService.findPackagesOnProcessing(); +// for (UserPackage userPackage : packages) { +// log.info("Updating flags for Package {}", userPackage.getPackageName()); +// process(userPackage.getId(), userPackage.getPackageName()); +// } +// log.info("Finish updating processing packages"); } private void process(Integer packageId, String packageName) { diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java index 5b8e1a6..829c480 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java @@ -1,12 +1,5 @@ package ru.nbch.credit_tracker.service.indic; -import java.time.LocalDate; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -15,6 +8,10 @@ import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; +import java.time.LocalDate; +import java.util.*; +import java.util.stream.Collectors; + @Service public class PhoneService { @@ -52,7 +49,7 @@ public class PhoneService { for (PhoneFidMap phoneFid : subjects) { while (iterator.hasNext()) { FidsData fidsData = iterator.next(); - if (fidsData.getPhone().equals(phoneFid.getPhone())) { + if (fidsData.getPhone().equals(9 + phoneFid.getPhone())) { //fixme доступ по ссылке, страшно, придумать что-то еще phoneFidMap.put(fidsData.getFid(), phoneFid); iterator.remove(); diff --git a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml index ccdde14..751fc7c 100644 --- a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml @@ -22,6 +22,7 @@ #{fid} + ) SELECT fid, CASE WHEN SUM(CASE WHEN amt_past_due > 5000 THEN 1 ELSE 0 END) > 0 From d136c3073910b346bd2b7e6c7b6017ee9ce09798 Mon Sep 17 00:00:00 2001 From: etreschenkov Date: Tue, 23 Sep 2025 14:14:49 +0300 Subject: [PATCH 03/20] collect fids for procession flag calc --- .../credit_tracker/service/SignalService.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index d5c80dc..f468fd1 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -114,6 +114,7 @@ public class SignalService { boolean noFidsFound = true; int offset = 0; List batch; + Map phonesByFidToProcessing = new HashMap<>(); do { batch = phoneFidMapService.findFullPhoneFidMap(packageId, offset, batchSize); @@ -129,15 +130,28 @@ public class SignalService { } if (!filteredItems.isEmpty()) { - boolean fidsFound = updateFlags(filteredItems, packageId); - if (noFidsFound) { - noFidsFound = !fidsFound; + Map fidsMapByBatch = phoneService.defineFidByPhone(filteredItems); + if (fidsMapByBatch.size() + phonesByFidToProcessing.size() < batchSize) { + phonesByFidToProcessing.putAll(fidsMapByBatch); + } else { + boolean fidsFound = updateFlags(phonesByFidToProcessing, packageId); + if (noFidsFound) { + noFidsFound = !fidsFound; + } + phonesByFidToProcessing = new HashMap<>(fidsMapByBatch); } } offset += batchSize; } while (!batch.isEmpty()); + if (!phonesByFidToProcessing.isEmpty()) { + boolean fidsFound = updateFlags(phonesByFidToProcessing, packageId); + if (noFidsFound) { + noFidsFound = !fidsFound; + } + } + return new UpdateResult(updated, noFidsFound); } @@ -146,9 +160,8 @@ public class SignalService { * @return fidsFound true - если хотя бы по одному номеру был найден фид, false - в противном случае */ // TODO Добавить отдельный метод. Если не нашелся фид, надо обновить в базе fid = null и флаги = null calculated_at=now() - public boolean updateFlags(List subjects, Integer packageId) { + public boolean updateFlags(Map fidsMap, Integer packageId) { log.info("Start calculating fids and flags for subjects batch. PackageId: {}", packageId); - Map fidsMap = phoneService.defineFidByPhone(subjects); Map flagsDataMap = flagsService.findFlags(fidsMap.keySet().stream().toList()); for (Map.Entry entry : flagsDataMap.entrySet()) { Integer fid = entry.getKey(); @@ -162,7 +175,7 @@ public class SignalService { phoneFidMapMapper.updatePhoneFid(phoneFidMap); } - log.info("Finish calculating fids and flags for subjects. PackageId: {}. Batch size: {}", packageId, subjects.size()); + log.info("Finish calculating fids and flags for subjects. PackageId: {}. Batch size: {}", packageId, fidsMap.size()); return !fidsMap.isEmpty(); } From 21bd541243c3658d168378b039eb74432aacac74 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Tue, 23 Sep 2025 14:20:57 +0300 Subject: [PATCH 04/20] Moved SignalServerException logs to GlobalExceptionHandler; Added warn logs to GlobalExceptionHandler for SignalClientException --- .../config/rest/GlobalExceptionHandler.java | 11 +++++- .../credit_tracker/enums/SignalError.java | 4 +-- .../exceptions/SignalServerException.java | 7 +++- .../credit_tracker/facade/SignalFacade.java | 34 ++++++------------- .../credit_tracker/utils/ValidationUtil.java | 32 +++++++++-------- .../utils/error/ExceptionUtils.java | 4 +-- 6 files changed, 47 insertions(+), 45 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java b/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java index 4ce54fe..2762bdb 100644 --- a/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java +++ b/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java @@ -1,5 +1,6 @@ package ru.nbch.credit_tracker.config.rest; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -12,10 +13,12 @@ import ru.nbch.credit_tracker.entities.user_response.errors.*; import ru.nbch.credit_tracker.enums.SignalError; import ru.nbch.credit_tracker.exceptions.SignalClientException; import ru.nbch.credit_tracker.exceptions.SignalException; +import ru.nbch.credit_tracker.exceptions.SignalServerException; import ru.nbch.credit_tracker.utils.error.SimpleMessageResolver; import java.nio.charset.Charset; +@Slf4j @RestControllerAdvice public class GlobalExceptionHandler { @@ -43,14 +46,20 @@ public class GlobalExceptionHandler { public ResponseEntity handleClientException(SignalException ex) { HttpHeaders headers = new HttpHeaders(); headers.setContentType(new MediaType("application", "xml", Charset.forName(encoding))); + + String errorText = messageResolver.resolve(ex.getErrorMessage()); + CommonError commonError = getCommonError(ex); commonError.setStatus("error"); commonError.setErrorCode(ex.getErrorMessage().getSubject().getCode()); - commonError.setErrorText(messageResolver.resolve(ex.getErrorMessage())); + commonError.setErrorText(errorText); if (ex instanceof SignalClientException) { + log.warn(errorText); return new ResponseEntity<>(commonError, headers, HttpStatus.BAD_REQUEST); } else { + Exception cause = ((SignalServerException) ex).getCause(); + log.error(cause.getMessage(), cause); return new ResponseEntity<>(commonError, headers, HttpStatus.INTERNAL_SERVER_ERROR); } } diff --git a/src/main/java/ru/nbch/credit_tracker/enums/SignalError.java b/src/main/java/ru/nbch/credit_tracker/enums/SignalError.java index 30f5e4c..ea68795 100644 --- a/src/main/java/ru/nbch/credit_tracker/enums/SignalError.java +++ b/src/main/java/ru/nbch/credit_tracker/enums/SignalError.java @@ -3,13 +3,13 @@ package ru.nbch.credit_tracker.enums; public enum SignalError implements IEnumKeyCode { ERROR_001("001", "Ошибка распаковки архива"), ERROR_002("002", "Отсутствует обязательный элемент: {}"), - ERROR_003("003", "Элемент ClientId не соответствует формату XXXXXXXXX"), + ERROR_003("003", "Элемент ClientId не соответствует формату XXXXXXXXX. Id: {}"), ERROR_005("005", "Неверный логин или пароль"), ERROR_006("006", "Доступ к услуге закрыт, обратитесь к курирующему менеджеру"), ERROR_007("007", "В системе уже существует пакет с таким именем"), ERROR_008("008", "PackageName не соответствует допустимому формату"), ERROR_009("009", "Превышен допустимый размер пакета"), - ERROR_010("010", "Идентификатор субъекта (Id) не соответствует допустимому формату"), + ERROR_010("010", "Идентификатор субъекта (Id) не соответствует допустимому формату: {}"), ERROR_011("011", "Получен невалидный XML"), ERROR_012("012", "Пакет {} не найден"), ERROR_013("013", "На стороне сервиса произошла ошибка, попробуйте повторить запрос позже или сообщите в техническую поддержку"), diff --git a/src/main/java/ru/nbch/credit_tracker/exceptions/SignalServerException.java b/src/main/java/ru/nbch/credit_tracker/exceptions/SignalServerException.java index 46dd713..dc1ef29 100644 --- a/src/main/java/ru/nbch/credit_tracker/exceptions/SignalServerException.java +++ b/src/main/java/ru/nbch/credit_tracker/exceptions/SignalServerException.java @@ -1,12 +1,17 @@ package ru.nbch.credit_tracker.exceptions; +import lombok.Getter; import ru.nbch.credit_tracker.enums.EnumKeyMessage; import ru.nbch.credit_tracker.enums.XmlErrorFormat; +@Getter public class SignalServerException extends SignalException { - public SignalServerException(EnumKeyMessage errorMessage, XmlErrorFormat errorFormat) { + private final Exception cause; + + public SignalServerException(EnumKeyMessage errorMessage, XmlErrorFormat errorFormat, Exception cause) { super(errorMessage, errorFormat); + this.cause = cause; } } diff --git a/src/main/java/ru/nbch/credit_tracker/facade/SignalFacade.java b/src/main/java/ru/nbch/credit_tracker/facade/SignalFacade.java index 218a1c8..ea6121d 100644 --- a/src/main/java/ru/nbch/credit_tracker/facade/SignalFacade.java +++ b/src/main/java/ru/nbch/credit_tracker/facade/SignalFacade.java @@ -109,21 +109,17 @@ public class SignalFacade { zipInputStream.getNextEntry(); filePath = writeTmpFile(zipInputStream); - if (filePath == null) { - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError); - } } catch (Exception e) { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e); } finally { if (zipInputStream != null) { try { zipInputStream.close(); } catch (IOException e) { - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e); } } } @@ -140,8 +136,7 @@ public class SignalFacade { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e); } try { @@ -165,8 +160,7 @@ public class SignalFacade { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e); } // в отдельном потоке запускаем регистрацию субъектов, поиск фидов и расчет флагов просрочек @@ -200,7 +194,7 @@ public class SignalFacade { return monitoringRequest; } - private String writeTmpFile(ZipInputStream zipInputStream) { + private String writeTmpFile(ZipInputStream zipInputStream) throws IOException { String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator; String filePath = folderPath + UUID.randomUUID() + "monitoring_request.xml"; CTUtil.createFolder(folderPath); @@ -214,9 +208,6 @@ public class SignalFacade { } return filePath; - } catch (Exception e) { - log.error("Ошибка при записи файла запроса постановки на мониторинг на диск", e); - return null; } } @@ -281,8 +272,7 @@ public class SignalFacade { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError, e); } } @@ -318,8 +308,7 @@ public class SignalFacade { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError, e); } } @@ -344,8 +333,7 @@ public class SignalFacade { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_017, XmlErrorFormat.MonitoringPackageDeleteError); + throw serverExcp(SignalError.ERROR_017, XmlErrorFormat.MonitoringPackageDeleteError, e); } } @@ -391,8 +379,7 @@ public class SignalFacade { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringReportListError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringReportListError, e); } } @@ -418,8 +405,7 @@ public class SignalFacade { if (e instanceof SignalClientException) { throw (SignalClientException) e; } - log.error(e.getMessage(), e); - throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringDownloadError); + throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringDownloadError, e); } } } diff --git a/src/main/java/ru/nbch/credit_tracker/utils/ValidationUtil.java b/src/main/java/ru/nbch/credit_tracker/utils/ValidationUtil.java index d0068b2..4885a27 100644 --- a/src/main/java/ru/nbch/credit_tracker/utils/ValidationUtil.java +++ b/src/main/java/ru/nbch/credit_tracker/utils/ValidationUtil.java @@ -1,5 +1,6 @@ package ru.nbch.credit_tracker.utils; +import lombok.extern.slf4j.Slf4j; import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest; import ru.nbch.credit_tracker.entities.user_request.monitoring_request.Subject; import ru.nbch.credit_tracker.enums.SignalError; @@ -10,25 +11,30 @@ import java.util.regex.Pattern; import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.clientExcp; +@Slf4j public class ValidationUtil { - private static String phoneRegex = "^\\d{9}$"; - private static String packageNameRegex = "^[A-Z0-9_]{1,22}$"; - private static String npIdRegex = "^\\S{1,36}$"; + private static final String phoneRegex = "^\\d{9}$"; + private static final String packageNameRegex = "^[A-Z0-9_]{1,22}$"; + private static final String npIdRegex = "^\\S{1,36}$"; public static void validateMonitoringRequest(MonitoringRequest request) throws SignalClientException { if (request == null) { throw clientExcp(SignalError.ERROR_011, XmlErrorFormat.MonitoringError); } - if (request.getUserId() == null) { - throw clientExcp(SignalError.ERROR_002, XmlErrorFormat.MonitoringError, "UserId"); - } - if (request.getPackageName() == null) { throw clientExcp(SignalError.ERROR_002, XmlErrorFormat.MonitoringError, "PackageName"); } + if (!validatePackageName(request.getPackageName())) { + throw clientExcp(SignalError.ERROR_008, XmlErrorFormat.MonitoringError); + } + + if (request.getUserId() == null) { + throw clientExcp(SignalError.ERROR_002, XmlErrorFormat.MonitoringError, "UserId"); + } + if (request.getPassword() == null) { throw clientExcp(SignalError.ERROR_002, XmlErrorFormat.MonitoringError, "Password"); } @@ -41,16 +47,12 @@ public class ValidationUtil { throw clientExcp(SignalError.ERROR_002, XmlErrorFormat.MonitoringError, "Subject"); } - if (!validatePackageName(request.getPackageName())) { - throw clientExcp(SignalError.ERROR_008, XmlErrorFormat.MonitoringError); - } - for (Subject subject : request.getSubjects().getSubject()) { - if (!validatePhone(subject.getClientId())) { - throw clientExcp(SignalError.ERROR_003, XmlErrorFormat.MonitoringError); - } if (!validateNPId(subject.getId())) { - throw clientExcp(SignalError.ERROR_010, XmlErrorFormat.MonitoringError); + throw clientExcp(SignalError.ERROR_010, XmlErrorFormat.MonitoringError, subject.getId()); + } + if (!validatePhone(subject.getClientId())) { + throw clientExcp(SignalError.ERROR_003, XmlErrorFormat.MonitoringError, subject.getId()); } } } diff --git a/src/main/java/ru/nbch/credit_tracker/utils/error/ExceptionUtils.java b/src/main/java/ru/nbch/credit_tracker/utils/error/ExceptionUtils.java index a15457c..2e94e78 100644 --- a/src/main/java/ru/nbch/credit_tracker/utils/error/ExceptionUtils.java +++ b/src/main/java/ru/nbch/credit_tracker/utils/error/ExceptionUtils.java @@ -12,7 +12,7 @@ public class ExceptionUtils { return new SignalClientException(new EnumKeyMessage(enumKeyCode, args), xmlErrorFormat); } - public static SignalServerException serverExcp(IEnumKeyCode enumKeyCode, XmlErrorFormat xmlErrorFormat, Object... args) { - return new SignalServerException(new EnumKeyMessage(enumKeyCode, args), xmlErrorFormat); + public static SignalServerException serverExcp(IEnumKeyCode enumKeyCode, XmlErrorFormat xmlErrorFormat, Exception cause, Object... args) { + return new SignalServerException(new EnumKeyMessage(enumKeyCode, args), xmlErrorFormat, cause); } } From a633a0d6054c60f633e3b474314124a6f19e52d8 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Tue, 23 Sep 2025 14:56:00 +0300 Subject: [PATCH 05/20] Fix tests; Added extra logs --- .../component/MapperTimingAspect.java | 2 +- .../config/rest/GlobalExceptionHandler.java | 2 +- .../credit_tracker/service/SignalService.java | 8 ++++-- src/main/resources/logback.xml | 4 +-- .../service/PhoneServiceTest.java | 28 ++++++++++--------- .../utils/ValidationUtilTest.java | 4 +-- 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java b/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java index e0f818a..05902b2 100644 --- a/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java +++ b/src/main/java/ru/nbch/credit_tracker/component/MapperTimingAspect.java @@ -18,7 +18,7 @@ public class MapperTimingAspect { } finally { long tookMs = (System.nanoTime() - start) / 1_000_000; String method = pjp.getSignature().toShortString(); - if (tookMs > 500) { + if (tookMs > 1000) { log.warn("Slow mapper call {} took {} ms", method, tookMs); } else { log.debug("Mapper call {} took {} ms", method, tookMs); diff --git a/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java b/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java index 2762bdb..b0824b1 100644 --- a/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java +++ b/src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java @@ -55,7 +55,7 @@ public class GlobalExceptionHandler { commonError.setErrorText(errorText); if (ex instanceof SignalClientException) { - log.warn(errorText); + log.warn("Response error: {}", errorText); return new ResponseEntity<>(commonError, headers, HttpStatus.BAD_REQUEST); } else { Exception cause = ((SignalServerException) ex).getCause(); diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index f468fd1..4306402 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -131,7 +131,9 @@ public class SignalService { if (!filteredItems.isEmpty()) { Map fidsMapByBatch = phoneService.defineFidByPhone(filteredItems); - if (fidsMapByBatch.size() + phonesByFidToProcessing.size() < batchSize) { + log.debug("Actual fids size by phone's batch: {}", fidsMapByBatch.size()); + + if (fidsMapByBatch.size() + phonesByFidToProcessing.size() <= batchSize) { phonesByFidToProcessing.putAll(fidsMapByBatch); } else { boolean fidsFound = updateFlags(phonesByFidToProcessing, packageId); @@ -161,7 +163,7 @@ public class SignalService { */ // TODO Добавить отдельный метод. Если не нашелся фид, надо обновить в базе fid = null и флаги = null calculated_at=now() public boolean updateFlags(Map fidsMap, Integer packageId) { - log.info("Start calculating fids and flags for subjects batch. PackageId: {}", packageId); + log.info("Start calculating flags for subjects batch. PackageId: {}", packageId); Map flagsDataMap = flagsService.findFlags(fidsMap.keySet().stream().toList()); for (Map.Entry entry : flagsDataMap.entrySet()) { Integer fid = entry.getKey(); @@ -175,7 +177,7 @@ public class SignalService { phoneFidMapMapper.updatePhoneFid(phoneFidMap); } - log.info("Finish calculating fids and flags for subjects. PackageId: {}. Batch size: {}", packageId, fidsMap.size()); + log.info("Finish calculating flags for subjects. PackageId: {}. Batch size: {}", packageId, fidsMap.size()); return !fidsMap.isEmpty(); } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 88ce91c..5b9cf16 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -63,12 +63,12 @@ - + - + diff --git a/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java b/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java index cba98b0..d552d1c 100644 --- a/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java +++ b/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java @@ -1,20 +1,22 @@ package ru.nbch.credit_tracker.service; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; +import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; +import ru.nbch.credit_tracker.service.indic.PhoneService; + import java.time.LocalDate; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; + import static org.mockito.Mockito.when; -import org.mockito.junit.jupiter.MockitoExtension; -import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; -import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; -import ru.nbch.credit_tracker.service.indic.PhoneService; @ExtendWith(MockitoExtension.class) public class PhoneServiceTest { @@ -43,10 +45,10 @@ public class PhoneServiceTest { private final List fidsDataWithEqualSinceDt = new ArrayList<>( List.of( - new PhoneService.FidsData(123, "1111", LocalDate.of(2025, 9, 10)), - new PhoneService.FidsData(124, "1111", LocalDate.of(2025, 9, 10)), - new PhoneService.FidsData(125, "1112", LocalDate.of(2024, 10, 10)), - new PhoneService.FidsData(126, "1114", LocalDate.of(2024, 10, 11)) + new PhoneService.FidsData(123, "91111", LocalDate.of(2025, 9, 10)), + new PhoneService.FidsData(124, "91111", LocalDate.of(2025, 9, 10)), + new PhoneService.FidsData(125, "91112", LocalDate.of(2024, 10, 10)), + new PhoneService.FidsData(126, "91114", LocalDate.of(2024, 10, 11)) ) ); diff --git a/src/test/java/ru/nbch/credit_tracker/utils/ValidationUtilTest.java b/src/test/java/ru/nbch/credit_tracker/utils/ValidationUtilTest.java index b8146f3..aea414b 100644 --- a/src/test/java/ru/nbch/credit_tracker/utils/ValidationUtilTest.java +++ b/src/test/java/ru/nbch/credit_tracker/utils/ValidationUtilTest.java @@ -69,12 +69,12 @@ public class ValidationUtilTest { Arguments.of( "/monitoring_request/invalid_phone_request.zip", SignalError.ERROR_003, - Collections.emptyList().toArray(new Object[0]) + new Object[]{"abc123"} ), Arguments.of( "/monitoring_request/invalid_id_request.zip", SignalError.ERROR_010, - Collections.emptyList().toArray(new Object[0]) + new Object[]{"#####"} ) ); } From 3f9fff5baaf7997278d8283710cc882256f3ca6f Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Tue, 23 Sep 2025 17:45:22 +0300 Subject: [PATCH 06/20] Fix PhoneService --- .../java/ru/nbch/credit_tracker/service/indic/PhoneService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java index 829c480..fe7c623 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java @@ -55,7 +55,6 @@ public class PhoneService { iterator.remove(); break; } - iterator.remove(); } } return phoneFidMap; From 669236e69689a856a5363e6e84c35c6e0b59b2d6 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Tue, 23 Sep 2025 18:18:48 +0300 Subject: [PATCH 07/20] Fix PhoneService --- .../ru/nbch/credit_tracker/service/indic/PhoneService.java | 7 ++++++- .../ru/nbch/credit_tracker/service/PhoneServiceTest.java | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java index fe7c623..fa3b27c 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java @@ -4,6 +4,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; @@ -12,6 +13,7 @@ import java.time.LocalDate; import java.util.*; import java.util.stream.Collectors; +@Slf4j @Service public class PhoneService { @@ -45,8 +47,9 @@ public class PhoneService { //отсортированный список по NUMBER, FID и FILE_SINCE_DT List phonesByFid = mapper.findFids(phones); - Iterator iterator = phonesByFid.iterator(); + log.debug("Start merging fids"); for (PhoneFidMap phoneFid : subjects) { + Iterator iterator = phonesByFid.iterator(); while (iterator.hasNext()) { FidsData fidsData = iterator.next(); if (fidsData.getPhone().equals(9 + phoneFid.getPhone())) { @@ -57,6 +60,8 @@ public class PhoneService { } } } + log.debug("End merging fids"); + return phoneFidMap; } diff --git a/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java b/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java index d552d1c..b3973e6 100644 --- a/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java +++ b/src/test/java/ru/nbch/credit_tracker/service/PhoneServiceTest.java @@ -34,6 +34,7 @@ public class PhoneServiceTest { List phoneFidMap = new ArrayList<>(); phoneFidMap.add(PhoneFidMap.builder().phone("1111").build()); phoneFidMap.add(PhoneFidMap.builder().phone("1112").build()); + phoneFidMap.add(PhoneFidMap.builder().phone("1113").build()); phoneFidMap.add(PhoneFidMap.builder().phone("1114").build()); Map res = phoneService.defineFidByPhone(phoneFidMap); From f083716071129442dbbc8ccd0d14d6ae75b8594c Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Wed, 24 Sep 2025 15:10:01 +0300 Subject: [PATCH 08/20] optimize updating --- .../credit_tracker/service/SignalService.java | 15 ++++++++++++++- src/main/resources/logback.xml | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index 4306402..0f3ca72 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -165,6 +165,7 @@ public class SignalService { public boolean updateFlags(Map fidsMap, Integer packageId) { log.info("Start calculating flags for subjects batch. PackageId: {}", packageId); Map flagsDataMap = flagsService.findFlags(fidsMap.keySet().stream().toList()); + List subjectToUpdate = new ArrayList<>(); for (Map.Entry entry : flagsDataMap.entrySet()) { Integer fid = entry.getKey(); FlagsService.FlagsData flagsData = entry.getValue(); @@ -175,8 +176,20 @@ public class SignalService { phoneFidMap.setFlag90days(flagsData.getFlag90days()); phoneFidMap.setCalculatedAt(LocalDateTime.now()); - phoneFidMapMapper.updatePhoneFid(phoneFidMap); + subjectToUpdate.add(phoneFidMap); } + + log.debug("Start updating subject's batch"); + if (!subjectToUpdate.isEmpty()) { + int updateBatchSize = 7000; // 9 параметров на одну запись для обновления. Максимальное количество параметров ~65000 + for (int j = 0; j < subjectToUpdate.size(); j += updateBatchSize) { + int endIndex = Math.min(j + updateBatchSize, subjectToUpdate.size()); + phoneFidMapService.updatePhoneFidMaps(subjectToUpdate.subList(j, endIndex)); + log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjectToUpdate.size(), packageId); + } + } + log.debug("Complete updating subject's batch"); + log.info("Finish calculating flags for subjects. PackageId: {}. Batch size: {}", packageId, fidsMap.size()); return !fidsMap.isEmpty(); } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 5b9cf16..20aa7c4 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -68,7 +68,7 @@ - + From 4114576a789828293ae3f6c0655b8a7a7d77f394 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Wed, 24 Sep 2025 15:10:24 +0300 Subject: [PATCH 09/20] all queries --- .../mapper/indic/FlagsMapper.java | 17 ++ .../service/indic/FlagsService.java | 70 ++++++++- .../mybatis/mapper/indic/FlagsMapper.xml | 147 ++++++++++++++++++ 3 files changed, 230 insertions(+), 4 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java index c64cd90..ecbce5f 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java @@ -16,4 +16,21 @@ public interface FlagsMapper { List findFlag90ByRuPaymt(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); + List findFlagsDebt5000Old(@Param("fids") List fids); + + List findFlag90ByPastDueArrearOld(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); + + List findFlag90ByRuPaymtOld(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); + + List findFlagsDebt5000New(@Param("fids") List fids); + + List findFlagsDebt5000NewWithOrderBy(@Param("fids") List fids); + + List findFlag90ByPastDueArrearNew(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); + + List findFlag90ByPastDueArrearNewWithOrderBy(@Param("fids") List fids); + + List findFlag90ByRuPaymtNew(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); + + List findFlag90ByRuPaymtNewWithOrderBy(@Param("fids") List fids); } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 5a11f68..17a6924 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -1,9 +1,5 @@ package ru.nbch.credit_tracker.service.indic; -import java.time.LocalDate; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; @@ -12,6 +8,13 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.mapper.indic.FlagsMapper; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + @Service public class FlagsService { @@ -29,8 +32,11 @@ public class FlagsService { List distinctFids = fids.stream().distinct().toList(); log.debug("Fids count: {}", distinctFids.size()); Map flags5000 = calcFlag5000(distinctFids); + calcFlag500Test(distinctFids); Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(distinctFids); + calcFlag90DaysByPastDueArrearTest(distinctFids); Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); + calcFlag90DaysByRuPaymtTest(distinctFids); Map result = new HashMap<>(); fids.forEach(fid -> { @@ -65,6 +71,18 @@ public class FlagsService { return res; } + private void calcFlag500Test(List fids) { + log.debug("Start findFlagsDebt5000Old"); + mapper.findFlagsDebt5000Old(fids); + log.debug("Complete findFlagsDebt5000Old"); + log.debug("Start findFlagsDebt5000New"); + mapper.findFlagsDebt5000New(fids); + log.debug("Complete findFlagsDebt5000New"); + log.debug("Start findFlagsDebt5000NewWithOrderBy"); + mapper.findFlagsDebt5000NewWithOrderBy(fids); + log.debug("Complete findFlagsDebt5000NewWithOrderBy"); + } + /** * Флаг = 1, если выполняется хотя бы одно из условий: * В INDIC.RU_PASTDUEARREAR или RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. @@ -79,6 +97,19 @@ public class FlagsService { return res; } + private void calcFlag90DaysByPastDueArrearTest(List fids) { + LocalDate yearAgo = LocalDate.now().minusYears(1); + log.debug("Start findFlag90ByPastDueArrearOld"); + mapper.findFlag90ByPastDueArrearOld(fids, yearAgo); + log.debug("Complete findFlag90ByPastDueArrearOld"); + log.debug("Start findFlag90ByPastDueArrearNew"); + mapper.findFlag90ByPastDueArrearNew(fids, yearAgo); + log.debug("Complete findFlag90ByPastDueArrearNew"); + log.debug("Start findFlag90ByPastDueArrearNewWithOrderBy"); + mapper.findFlag90ByPastDueArrearNewWithOrderBy(fids); + log.debug("Complete findFlag90ByPastDueArrearNewWithOrderBy"); + } + private Map calcFlag90DaysByRuPaymt(List fids) { log.debug("Start calculating flag 90 by ruPaymt"); LocalDate yearAgo = LocalDate.now().minusYears(1); @@ -88,6 +119,19 @@ public class FlagsService { return res; } + private void calcFlag90DaysByRuPaymtTest(List fids) { + LocalDate yearAgo = LocalDate.now().minusYears(1); + log.debug("Start findFlag90ByRuPaymtOld"); + mapper.findFlag90ByRuPaymtOld(fids,yearAgo); + log.debug("Complete findFlag90ByRuPaymtOld"); + log.debug("Start findFlag90ByRuPaymtNew"); + mapper.findFlag90ByRuPaymtNew(fids, yearAgo); + log.debug("Complete findFlag90ByRuPaymtNew"); + log.debug("Start findFlag90ByRuPaymtNewWithOrderBy"); + mapper.findFlag90ByRuPaymtNewWithOrderBy(fids); + log.debug("Complete findFlag90ByRuPaymtNewWithOrderBy"); + } + private Map makeFidFlagMapByRs(List fids, List rows) { Map result = new HashMap<>(); for (Integer fid : fids) { @@ -98,6 +142,7 @@ public class FlagsService { } return result; } + @Getter @Setter @AllArgsConstructor @@ -112,4 +157,21 @@ public class FlagsService { private Integer fid; private Integer flag; } + + @Getter + @Setter + public static class Debt5000Data { + private Integer fid; + private BigInteger accSerialNum; + private LocalDate reportedDate; + private BigDecimal amtPastDue; + } + + @Getter + @Setter + public static class Flag90DaysData { + private Integer fid; + private BigInteger daysPastDue; + private LocalDate reportedDate; + } } diff --git a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml index 751fc7c..410ab04 100644 --- a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml @@ -78,4 +78,151 @@ GROUP BY fid WITH UR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 021d7e861570784ab572eafc6ed04dd50103668c Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Wed, 24 Sep 2025 15:59:04 +0300 Subject: [PATCH 10/20] Removed order by reported_dt --- .../mapper/indic/FlagsMapper.java | 9 --------- .../service/indic/FlagsService.java | 20 ------------------- .../mybatis/mapper/indic/FlagsMapper.xml | 6 +++--- 3 files changed, 3 insertions(+), 32 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java index ecbce5f..3ab6e2b 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java @@ -16,21 +16,12 @@ public interface FlagsMapper { List findFlag90ByRuPaymt(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); - List findFlagsDebt5000Old(@Param("fids") List fids); - - List findFlag90ByPastDueArrearOld(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); - - List findFlag90ByRuPaymtOld(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); - - List findFlagsDebt5000New(@Param("fids") List fids); List findFlagsDebt5000NewWithOrderBy(@Param("fids") List fids); - List findFlag90ByPastDueArrearNew(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); List findFlag90ByPastDueArrearNewWithOrderBy(@Param("fids") List fids); - List findFlag90ByRuPaymtNew(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); List findFlag90ByRuPaymtNewWithOrderBy(@Param("fids") List fids); } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 17a6924..783ec5c 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -72,12 +72,6 @@ public class FlagsService { } private void calcFlag500Test(List fids) { - log.debug("Start findFlagsDebt5000Old"); - mapper.findFlagsDebt5000Old(fids); - log.debug("Complete findFlagsDebt5000Old"); - log.debug("Start findFlagsDebt5000New"); - mapper.findFlagsDebt5000New(fids); - log.debug("Complete findFlagsDebt5000New"); log.debug("Start findFlagsDebt5000NewWithOrderBy"); mapper.findFlagsDebt5000NewWithOrderBy(fids); log.debug("Complete findFlagsDebt5000NewWithOrderBy"); @@ -98,13 +92,6 @@ public class FlagsService { } private void calcFlag90DaysByPastDueArrearTest(List fids) { - LocalDate yearAgo = LocalDate.now().minusYears(1); - log.debug("Start findFlag90ByPastDueArrearOld"); - mapper.findFlag90ByPastDueArrearOld(fids, yearAgo); - log.debug("Complete findFlag90ByPastDueArrearOld"); - log.debug("Start findFlag90ByPastDueArrearNew"); - mapper.findFlag90ByPastDueArrearNew(fids, yearAgo); - log.debug("Complete findFlag90ByPastDueArrearNew"); log.debug("Start findFlag90ByPastDueArrearNewWithOrderBy"); mapper.findFlag90ByPastDueArrearNewWithOrderBy(fids); log.debug("Complete findFlag90ByPastDueArrearNewWithOrderBy"); @@ -120,13 +107,6 @@ public class FlagsService { } private void calcFlag90DaysByRuPaymtTest(List fids) { - LocalDate yearAgo = LocalDate.now().minusYears(1); - log.debug("Start findFlag90ByRuPaymtOld"); - mapper.findFlag90ByRuPaymtOld(fids,yearAgo); - log.debug("Complete findFlag90ByRuPaymtOld"); - log.debug("Start findFlag90ByRuPaymtNew"); - mapper.findFlag90ByRuPaymtNew(fids, yearAgo); - log.debug("Complete findFlag90ByRuPaymtNew"); log.debug("Start findFlag90ByRuPaymtNewWithOrderBy"); mapper.findFlag90ByRuPaymtNewWithOrderBy(fids); log.debug("Complete findFlag90ByRuPaymtNewWithOrderBy"); diff --git a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml index 410ab04..4ad119e 100644 --- a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml @@ -171,7 +171,7 @@ #{fid} - ORDER BY FID, ACC_SERIAL_NUM, REPORTED_DT DESC + ORDER BY FID DESC with UR @@ -199,7 +199,7 @@ #{fid} - ORDER BY FID, REPORTED_DT DESC + ORDER BY FID DESC WITH UR @@ -221,7 +221,7 @@ #{fid} - ORDER BY FID, REPORTED_DT DESC + ORDER BY FID DESC WITH UR From c245790eb5c5e945bb8733fb109391962dd7266f Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Thu, 25 Sep 2025 13:07:39 +0300 Subject: [PATCH 11/20] Added test variant of counting flags 500 and 90_days from RU_PASTDUEARREAR --- credit-tracker.properties | 6 +- .../CreditTrackerApplication.java | 28 ++- .../mapper/indic/FlagsMapper.java | 6 +- .../service/indic/FlagsService.java | 113 +++++++++--- .../mybatis/mapper/indic/FlagsMapper.xml | 164 ++---------------- .../service/FlagsServiceTest.java | 152 ++++++++++++++++ 6 files changed, 281 insertions(+), 188 deletions(-) create mode 100644 src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java diff --git a/credit-tracker.properties b/credit-tracker.properties index a337a0e..c9fdfd6 100644 --- a/credit-tracker.properties +++ b/credit-tracker.properties @@ -8,9 +8,9 @@ server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b # datasources # indic ct.indic-data-source-settings.driver=com.ibm.db2.jcc.DB2Driver -ct.indic-data-source-settings.url=${DB_URL_INDIC:jdbc:db2://10.230.227.101:2668/cprosd22:currentSchema=INDIC;} -ct.indic-data-source-settings.username=${USERNAME_INDIC:tester} -ct.indic-data-source-settings.password=${PASSWORD_INDIC:tester} +ct.indic-data-source-settings.url=${DB_URL_INDIC:jdbc:db2://10.230.227.100:2668/cprosd22:currentSchema=INDIC;} +ct.indic-data-source-settings.username=${USERNAME_INDIC:ros9appl} +ct.indic-data-source-settings.password=${PASSWORD_INDIC:R0s9Appl} # signal ct.signal-data-source-settings.driver=org.postgresql.Driver diff --git a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java index d3a78f1..b30948b 100644 --- a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java +++ b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java @@ -3,21 +3,31 @@ package ru.nbch.credit_tracker; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; -import ru.nbch.credit_tracker.service.MonitoringService; -import ru.nbch.credit_tracker.service.UpdateProcessingPackagesService; -import ru.nbch.credit_tracker.service.UpdatingFlagsService; +import ru.nbch.credit_tracker.service.indic.FlagsService; + +import java.util.ArrayList; +import java.util.List; @SpringBootApplication public class CreditTrackerApplication { public static void main(String[] args) { ConfigurableApplicationContext ctx = SpringApplication.run(CreditTrackerApplication.class, args); - UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class); - MonitoringService monitoringService = ctx.getBean(MonitoringService.class); - UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class); - updateProcessingPackagesService.run(); - monitoringService.run(); - updatingFlagsService.run(); + List fids = new ArrayList<>(); + fids.add(11085548); + fids.add(16442833); + fids.add(109128766); + fids.add(115150769); + fids.add(16442929); + fids.add(114429070); + fids.add(108696892); + ctx.getBean(FlagsService.class).test(fids); +// UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class); +// MonitoringService monitoringService = ctx.getBean(MonitoringService.class); +// UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class); +// updateProcessingPackagesService.run(); +// monitoringService.run(); +// updatingFlagsService.run(); } } diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java index 3ab6e2b..1a162a7 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java @@ -16,11 +16,7 @@ public interface FlagsMapper { List findFlag90ByRuPaymt(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); - - List findFlagsDebt5000NewWithOrderBy(@Param("fids") List fids); - - - List findFlag90ByPastDueArrearNewWithOrderBy(@Param("fids") List fids); + List findRuPastDuarearData(@Param("fids") List fids); List findFlag90ByRuPaymtNewWithOrderBy(@Param("fids") List fids); diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 783ec5c..bb5d848 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -2,6 +2,7 @@ package ru.nbch.credit_tracker.service.indic; import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,9 +12,7 @@ import ru.nbch.credit_tracker.mapper.indic.FlagsMapper; import java.math.BigDecimal; import java.math.BigInteger; import java.time.LocalDate; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Service public class FlagsService { @@ -25,6 +24,88 @@ public class FlagsService { this.mapper = mapper; } + public void test(List fids) { + Map oldFlags5000 = calcFlag5000(fids); + Map oldFlags90 = calcFlag90DaysByPastDueArrear(fids); + + fids.sort(Comparator.naturalOrder()); + + Map flag5000Map = new HashMap<>(); + Map flag90Days = new HashMap<>(); + findFlags5000(fids, flag5000Map, flag90Days); + + for (Map.Entry flag : flag5000Map.entrySet()) { + if(!flag.getValue().equals(oldFlags5000.get(flag.getKey()))) { + String error = "New: "+flag.getKey() + " = " + flag.getValue() + "\n" +"Old: " + flag.getKey() + " = " + oldFlags5000.get(flag.getKey()); + throw new AssertionError(error); + } + } + + for (Map.Entry flag : flag90Days.entrySet()) { + if(!flag.getValue().equals(oldFlags90.get(flag.getKey()))) { + String error = "New: "+flag.getKey() + " = " + flag.getValue() + "\n" +"Old: " + flag.getKey() + " = " + oldFlags90.get(flag.getKey()); + throw new AssertionError(error); + } + } + + } + + private void findFlags5000(List fids, Map flag5000Map, Map flag90Days) { + List ruPastDuarearDataList = mapper.findRuPastDuarearData(fids); + ruPastDuarearDataList.sort(Comparator.comparing(FlagsService.RuPastDuarearData::getFid) // сортируем сначала по фиду + .thenComparing(FlagsService.RuPastDuarearData::getReportedDate, Comparator.reverseOrder())); // Затем сортируем по REPORTED_DT по убыванию + + LocalDate yearAgo = LocalDate.parse("2024-09-09"); + + for (Integer fid : fids) { + Iterator iterator = ruPastDuarearDataList.iterator(); + BigInteger prevAccSerialNum = null; + while (iterator.hasNext()) { + FlagsService.RuPastDuarearData data = iterator.next(); + + if (fid.equals(data.getFid())) { + if (flag5000Map.get(fid) == null) { // Первая запись в списке с макс REPORTED_DT. Нас интересует только она для флага 5000 + if (prevAccSerialNum == null || !prevAccSerialNum.equals(data.getAccSerialNum())) { + if (data.getAmtPastDue() != null && data.getAmtPastDue().compareTo(BigDecimal.valueOf(5000)) >= 0) { + flag5000Map.put(fid, 1); + } else { + flag5000Map.put(fid, 0); + } + prevAccSerialNum = data.getAccSerialNum(); + } + } + + if (flag90Days.get(fid) != null && flag90Days.get(fid) == 1) { + iterator.remove(); + break; + } + + if (flag90Days.get(fid) == null || flag90Days.get(fid) == 0) { + if (data.getDaysPastDue() != null) { + boolean has90daysFlag = data.getDaysPastDue().compareTo(BigInteger.valueOf(90)) >= 0 && !data.getReportedDate().isBefore(yearAgo); + if (has90daysFlag) { + flag90Days.put(fid, 1); + iterator.remove(); + break; + } else { + flag90Days.put(fid, 0); + iterator.remove(); + } + } else { + iterator.remove(); + } + } + } + } + if (flag5000Map.get(fid) == null) { + flag5000Map.put(fid, 0); + } + if (flag90Days.get(fid) == null) { + flag90Days.put(fid, 0); + } + } + } + public Map findFlags(List fids) { if (fids.isEmpty()) { return new HashMap<>(); @@ -32,11 +113,8 @@ public class FlagsService { List distinctFids = fids.stream().distinct().toList(); log.debug("Fids count: {}", distinctFids.size()); Map flags5000 = calcFlag5000(distinctFids); - calcFlag500Test(distinctFids); Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(distinctFids); - calcFlag90DaysByPastDueArrearTest(distinctFids); Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); - calcFlag90DaysByRuPaymtTest(distinctFids); Map result = new HashMap<>(); fids.forEach(fid -> { @@ -71,12 +149,6 @@ public class FlagsService { return res; } - private void calcFlag500Test(List fids) { - log.debug("Start findFlagsDebt5000NewWithOrderBy"); - mapper.findFlagsDebt5000NewWithOrderBy(fids); - log.debug("Complete findFlagsDebt5000NewWithOrderBy"); - } - /** * Флаг = 1, если выполняется хотя бы одно из условий: * В INDIC.RU_PASTDUEARREAR или RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. @@ -91,12 +163,6 @@ public class FlagsService { return res; } - private void calcFlag90DaysByPastDueArrearTest(List fids) { - log.debug("Start findFlag90ByPastDueArrearNewWithOrderBy"); - mapper.findFlag90ByPastDueArrearNewWithOrderBy(fids); - log.debug("Complete findFlag90ByPastDueArrearNewWithOrderBy"); - } - private Map calcFlag90DaysByRuPaymt(List fids) { log.debug("Start calculating flag 90 by ruPaymt"); LocalDate yearAgo = LocalDate.now().minusYears(1); @@ -106,12 +172,6 @@ public class FlagsService { return res; } - private void calcFlag90DaysByRuPaymtTest(List fids) { - log.debug("Start findFlag90ByRuPaymtNewWithOrderBy"); - mapper.findFlag90ByRuPaymtNewWithOrderBy(fids); - log.debug("Complete findFlag90ByRuPaymtNewWithOrderBy"); - } - private Map makeFidFlagMapByRs(List fids, List rows) { Map result = new HashMap<>(); for (Integer fid : fids) { @@ -140,11 +200,14 @@ public class FlagsService { @Getter @Setter - public static class Debt5000Data { + @AllArgsConstructor + @NoArgsConstructor + public static class RuPastDuarearData { private Integer fid; private BigInteger accSerialNum; private LocalDate reportedDate; private BigDecimal amtPastDue; + private BigInteger daysPastDue; } @Getter diff --git a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml index 4ad119e..ab3bb98 100644 --- a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml @@ -10,6 +10,24 @@ + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java b/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java new file mode 100644 index 0000000..41fb346 --- /dev/null +++ b/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java @@ -0,0 +1,152 @@ +package ru.nbch.credit_tracker.service; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import ru.nbch.credit_tracker.mapper.indic.FlagsMapper; +import ru.nbch.credit_tracker.service.indic.FlagsService; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.util.*; + +@ExtendWith(MockitoExtension.class) +public class FlagsServiceTest { + + @Mock + FlagsMapper flagsMapper; + + @Test + public void test() { + List fids = new ArrayList<>(List.of( + 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 2, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Нет просрочки 90_days в RU_PAYMT + 3, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 4, // нет никаких просрочек + 7, // нет просрочки 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 5, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 6, // нет просрочки 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 8, // нету в RU_PASTDUEARREAR, есть в RU_PAYMT + 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT + ); + + fids.sort(Comparator.naturalOrder()); + + Map flag5000Map = new HashMap<>(); + Map flag90Days = new HashMap<>(); + findFlags5000(fids, flag5000Map, flag90Days); + + Assertions.assertEquals(9, flag5000Map.size()); + Assertions.assertEquals(true, flag5000Map.get(1)); + Assertions.assertEquals(true, flag5000Map.get(2)); + Assertions.assertEquals(true, flag5000Map.get(3)); + Assertions.assertEquals(false, flag5000Map.get(4)); + Assertions.assertEquals(true, flag5000Map.get(5)); + Assertions.assertEquals(false, flag5000Map.get(6)); + Assertions.assertEquals(false, flag5000Map.get(7)); + Assertions.assertEquals(false, flag5000Map.get(8)); + Assertions.assertEquals(false, flag5000Map.get(9)); + + Assertions.assertEquals(9, flag90Days.size()); + Assertions.assertEquals(true, flag90Days.get(1)); + Assertions.assertEquals(false, flag90Days.get(2)); + Assertions.assertEquals(false, flag90Days.get(3)); + Assertions.assertEquals(false, flag90Days.get(4)); + Assertions.assertEquals(true, flag90Days.get(5)); + Assertions.assertEquals(false, flag90Days.get(6)); + Assertions.assertEquals(true, flag90Days.get(7)); + Assertions.assertEquals(false, flag90Days.get(8)); + Assertions.assertEquals(false, flag90Days.get(9)); + } + + private void findFlags5000(List fids, Map flag5000Map, Map flag90Days) { + ruPastDuarearDataList.sort(Comparator.comparing(FlagsService.RuPastDuarearData::getFid) // сортируем сначала по фиду + .thenComparing(FlagsService.RuPastDuarearData::getReportedDate, Comparator.reverseOrder())); // Затем сортируем по REPORTED_DT по убыванию + + LocalDate yearAgo = LocalDate.parse("2024-09-09"); + + for (Integer fid : fids) { + Iterator iterator = ruPastDuarearDataList.iterator(); + BigInteger prevAccSerialNum = null; + while (iterator.hasNext()) { + FlagsService.RuPastDuarearData data = iterator.next(); + + if (fid.equals(data.getFid())) { + if (flag5000Map.get(fid) == null) { // Первая запись в списке с макс REPORTED_DT. Нас интересует только она для флага 5000 + if (prevAccSerialNum == null || !prevAccSerialNum.equals(data.getAccSerialNum())) { + if (data.getAmtPastDue() != null && data.getAmtPastDue().compareTo(BigDecimal.valueOf(5000)) >= 0) { + flag5000Map.put(fid, true); + } else { + flag5000Map.put(fid, false); + } + prevAccSerialNum = data.getAccSerialNum(); + } + } + + if (flag90Days.get(fid) != null && flag90Days.get(fid).equals(Boolean.TRUE)) { + iterator.remove(); + break; + } + + if (flag90Days.get(fid) == null || flag90Days.get(fid).equals(Boolean.FALSE)) { + if (data.getDaysPastDue() != null) { + boolean has90daysFlag = data.getDaysPastDue().compareTo(BigInteger.valueOf(90)) >= 0 && !data.getReportedDate().isBefore(yearAgo); + if (has90daysFlag) { + flag90Days.put(fid, true); + iterator.remove(); + break; + } else { + flag90Days.put(fid, false); + iterator.remove(); + } + } else { + iterator.remove(); + } + } + } + } + if (flag5000Map.get(fid) == null) { + flag5000Map.put(fid, false); + } + if (flag90Days.get(fid) == null) { + flag90Days.put(fid, false); + } + } + } + + private final List ruPastDuarearDataList = new ArrayList<>( + List.of( + new FlagsService.RuPastDuarearData(1, BigInteger.valueOf(1), LocalDate.parse("2025-12-31"), BigDecimal.valueOf(36000.00), BigInteger.valueOf(159)), // просрочка 5000 и 90_days по этой записи + + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2021-12-29"), BigDecimal.valueOf(3200.00), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2022-01-28"), BigDecimal.valueOf(6700.00), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2025-04-06"), BigDecimal.valueOf(5700.00), null), // просрочка 5000 по этой записи + + new FlagsService.RuPastDuarearData(3, BigInteger.valueOf(3), LocalDate.parse("2025-01-14"), BigDecimal.valueOf(300000000000.00), null), + + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2021-11-29"), BigDecimal.valueOf(0.00), null), // не макс reported_date для того же acc_serial_num + + new FlagsService.RuPastDuarearData(4, BigInteger.valueOf(4), LocalDate.parse("2025-12-06"), BigDecimal.valueOf(4000.00), null), // нет просрочек + + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-04-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(182)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-05-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(212)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-05-31"), BigDecimal.valueOf(2000.31), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-07-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(273)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-07-31"), BigDecimal.valueOf(2000.31), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-08-04"), BigDecimal.valueOf(7000.31), BigInteger.valueOf(307)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2025-09-08"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(338)), // не макс reported_date для того же acc_serial_num, просрочка 90_days будет взята у этой записи + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(51), LocalDate.parse("2025-09-09"), BigDecimal.valueOf(7000.31), BigInteger.valueOf(368)), // Просрочка 5000 будет взята у этой записи. Уникальный acc_serial_num с макс reported_dt + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-11-04"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(399)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2022-01-04"), BigDecimal.valueOf(5000.31), BigInteger.valueOf(5)), // не макс reported_date для того же acc_serial_num + + new FlagsService.RuPastDuarearData(6, BigInteger.valueOf(6), LocalDate.parse("2025-09-09"), BigDecimal.valueOf(3000), null), // Нет просрочки 5000, нет просрочки 90_days (90_days будет в RU_PAYMT) + + new FlagsService.RuPastDuarearData(7, BigInteger.valueOf(7), LocalDate.parse("2025-09-09"), BigDecimal.valueOf(5500), BigInteger.valueOf(90)), // не макс reported_date для того же acc_serial_num. 90_days будет взята отсюда + new FlagsService.RuPastDuarearData(7, BigInteger.valueOf(7), LocalDate.parse("2025-09-10"), BigDecimal.valueOf(3000), null) // Нет просрочки 5000. Уникальный acc_serial_num с макс reported_dt + + ) + ); +} From b42e4f70df05464d2827cab3aed58bf998e47910 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Thu, 25 Sep 2025 15:46:04 +0300 Subject: [PATCH 12/20] Added finding flags for RU_PASTDUEARREAR; Updated tests --- .../CreditTrackerApplication.java | 14 +- .../mapper/indic/FlagsMapper.java | 2 - .../service/indic/FlagsService.java | 117 +++++++------ .../service/FlagsServiceTest.java | 164 +++++++----------- 4 files changed, 140 insertions(+), 157 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java index b30948b..fa2c9f9 100644 --- a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java +++ b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java @@ -7,6 +7,7 @@ import ru.nbch.credit_tracker.service.indic.FlagsService; import java.util.ArrayList; import java.util.List; +import java.util.Map; @SpringBootApplication public class CreditTrackerApplication { @@ -21,7 +22,18 @@ public class CreditTrackerApplication { fids.add(16442929); fids.add(114429070); fids.add(108696892); - ctx.getBean(FlagsService.class).test(fids); + FlagsService flagService = ctx.getBean(FlagsService.class); + Map oldMap = flagService.findFlags(fids); + Map newMap = flagService.findFlags2(fids); + System.out.println(oldMap); + System.out.println(newMap); + for (Integer fid : fids) { + FlagsService.FlagsData oldValue = oldMap.get(fid); + FlagsService.FlagsData newValue = newMap.get(fid); + System.out.println("OLD: " + oldValue.getFlagDebt5000() +"|" +oldValue.getFlag90days()); + System.out.println("NEW: " + newValue.getFlagDebt5000() +"|" +newValue.getFlag90days()); + System.out.println("_________________________________"); + } // UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class); // MonitoringService monitoringService = ctx.getBean(MonitoringService.class); // UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class); diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java index 1a162a7..1aa12be 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java @@ -18,6 +18,4 @@ public interface FlagsMapper { List findRuPastDuarearData(@Param("fids") List fids); - - List findFlag90ByRuPaymtNewWithOrderBy(@Param("fids") List fids); } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index bb5d848..209cce3 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -24,45 +24,85 @@ public class FlagsService { this.mapper = mapper; } - public void test(List fids) { - Map oldFlags5000 = calcFlag5000(fids); - Map oldFlags90 = calcFlag90DaysByPastDueArrear(fids); - - fids.sort(Comparator.naturalOrder()); - - Map flag5000Map = new HashMap<>(); - Map flag90Days = new HashMap<>(); - findFlags5000(fids, flag5000Map, flag90Days); - - for (Map.Entry flag : flag5000Map.entrySet()) { - if(!flag.getValue().equals(oldFlags5000.get(flag.getKey()))) { - String error = "New: "+flag.getKey() + " = " + flag.getValue() + "\n" +"Old: " + flag.getKey() + " = " + oldFlags5000.get(flag.getKey()); - throw new AssertionError(error); - } + public Map findFlags(List fids) { + if (fids.isEmpty()) { + return new HashMap<>(); } + List distinctFids = fids.stream().distinct().toList(); + log.debug("Fids count: {}", distinctFids.size()); + Map flags5000 = calcFlag5000(distinctFids); + Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(distinctFids); + Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); - for (Map.Entry flag : flag90Days.entrySet()) { - if(!flag.getValue().equals(oldFlags90.get(flag.getKey()))) { - String error = "New: "+flag.getKey() + " = " + flag.getValue() + "\n" +"Old: " + flag.getKey() + " = " + oldFlags90.get(flag.getKey()); - throw new AssertionError(error); + Map result = new HashMap<>(); + fids.forEach(fid -> { + Integer flagDebt5000 = flags5000.get(fid); + if (flagDebt5000 == null) { + flagDebt5000 = 0; } - } + Integer flag90days = flags90DaysByPastDueArrear.get(fid); + if (flag90days == null || flag90days == 0) { + flag90days = flags90DaysByRuPaymt.get(fid); + if (flag90days == null) { + flag90days = 0; + } + } + result.put(fid, new FlagsData(flagDebt5000, flag90days)); + }); + + return result; } - private void findFlags5000(List fids, Map flag5000Map, Map flag90Days) { + public Map findFlags2(List fids) { + if (fids.isEmpty()) { + return new HashMap<>(); + } + List distinctFids = fids.stream() + .distinct() + .sorted(Comparator.naturalOrder()) + .toList(); + + log.debug("Fids count: {}", distinctFids.size()); + Map flags5000 = new HashMap<>(); + Map flags90DaysByPastDueArrear = new HashMap<>(); + findFlagsRuPastDueArrear(distinctFids, flags5000, flags90DaysByPastDueArrear); + Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); + + Map result = new HashMap<>(); + fids.forEach(fid -> { + Integer flagDebt5000 = flags5000.get(fid); + if (flagDebt5000 == null) { + flagDebt5000 = 0; + } + + Integer flag90days = flags90DaysByPastDueArrear.get(fid); + if (flag90days == null || flag90days == 0) { + flag90days = flags90DaysByRuPaymt.get(fid); + if (flag90days == null) { + flag90days = 0; + } + } + result.put(fid, new FlagsData(flagDebt5000, flag90days)); + }); + + return result; + } + + private void findFlagsRuPastDueArrear(List fids, Map flag5000Map, Map flag90Days) { + log.debug("Start calculating flag 5000 and 90 days by RU_PASTDUEARREAR"); List ruPastDuarearDataList = mapper.findRuPastDuarearData(fids); + ruPastDuarearDataList.sort(Comparator.comparing(FlagsService.RuPastDuarearData::getFid) // сортируем сначала по фиду .thenComparing(FlagsService.RuPastDuarearData::getReportedDate, Comparator.reverseOrder())); // Затем сортируем по REPORTED_DT по убыванию - LocalDate yearAgo = LocalDate.parse("2024-09-09"); + LocalDate yearAgo = LocalDate.now().minusYears(1); for (Integer fid : fids) { Iterator iterator = ruPastDuarearDataList.iterator(); BigInteger prevAccSerialNum = null; while (iterator.hasNext()) { FlagsService.RuPastDuarearData data = iterator.next(); - if (fid.equals(data.getFid())) { if (flag5000Map.get(fid) == null) { // Первая запись в списке с макс REPORTED_DT. Нас интересует только она для флага 5000 if (prevAccSerialNum == null || !prevAccSerialNum.equals(data.getAccSerialNum())) { @@ -104,36 +144,7 @@ public class FlagsService { flag90Days.put(fid, 0); } } - } - - public Map findFlags(List fids) { - if (fids.isEmpty()) { - return new HashMap<>(); - } - List distinctFids = fids.stream().distinct().toList(); - log.debug("Fids count: {}", distinctFids.size()); - Map flags5000 = calcFlag5000(distinctFids); - Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(distinctFids); - Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); - - Map result = new HashMap<>(); - fids.forEach(fid -> { - Integer flagDebt5000 = flags5000.get(fid); - if (flagDebt5000 == null) { - flagDebt5000 = 0; - } - - Integer flag90days = flags90DaysByPastDueArrear.get(fid); - if (flag90days == null) { - flag90days = flags90DaysByRuPaymt.get(fid); - if (flag90days == null) { - flag90days = 0; - } - } - result.put(fid, new FlagsData(flagDebt5000, flag90days)); - }); - - return result; + log.debug("Complete calculating flag 5000 and 90 days by RU_PASTDUEARREAR"); } /** diff --git a/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java b/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java index 41fb346..3191a7c 100644 --- a/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java +++ b/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import ru.nbch.credit_tracker.mapper.indic.FlagsMapper; import ru.nbch.credit_tracker.service.indic.FlagsService; @@ -11,16 +12,30 @@ import ru.nbch.credit_tracker.service.indic.FlagsService; import java.math.BigDecimal; import java.math.BigInteger; import java.time.LocalDate; -import java.util.*; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class FlagsServiceTest { + private static final LocalDate INITIAL_DATE = LocalDate.parse("2025-09-25"); + @Mock FlagsMapper flagsMapper; @Test - public void test() { + public void testfindFlagsWithoutRuPaymt() { + + FlagsService flagsService = new FlagsService(flagsMapper); + + when(flagsMapper.findRuPastDuarearData(Mockito.anyList())).thenReturn(ruPastDuarearDataList); + when(flagsMapper.findFlag90ByRuPaymt(Mockito.anyList(), Mockito.any())).thenReturn(new ArrayList<>()); + List fids = new ArrayList<>(List.of( 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR 2, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Нет просрочки 90_days в RU_PAYMT @@ -33,120 +48,67 @@ public class FlagsServiceTest { 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT ); - fids.sort(Comparator.naturalOrder()); + Map resultMap = flagsService.findFlags2(fids); - Map flag5000Map = new HashMap<>(); - Map flag90Days = new HashMap<>(); - findFlags5000(fids, flag5000Map, flag90Days); + Assertions.assertEquals(9, resultMap.size()); + Assertions.assertEquals(1, resultMap.get(1).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(2).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(3).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(4).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(5).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(6).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(7).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(8).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(9).getFlagDebt5000()); - Assertions.assertEquals(9, flag5000Map.size()); - Assertions.assertEquals(true, flag5000Map.get(1)); - Assertions.assertEquals(true, flag5000Map.get(2)); - Assertions.assertEquals(true, flag5000Map.get(3)); - Assertions.assertEquals(false, flag5000Map.get(4)); - Assertions.assertEquals(true, flag5000Map.get(5)); - Assertions.assertEquals(false, flag5000Map.get(6)); - Assertions.assertEquals(false, flag5000Map.get(7)); - Assertions.assertEquals(false, flag5000Map.get(8)); - Assertions.assertEquals(false, flag5000Map.get(9)); - - Assertions.assertEquals(9, flag90Days.size()); - Assertions.assertEquals(true, flag90Days.get(1)); - Assertions.assertEquals(false, flag90Days.get(2)); - Assertions.assertEquals(false, flag90Days.get(3)); - Assertions.assertEquals(false, flag90Days.get(4)); - Assertions.assertEquals(true, flag90Days.get(5)); - Assertions.assertEquals(false, flag90Days.get(6)); - Assertions.assertEquals(true, flag90Days.get(7)); - Assertions.assertEquals(false, flag90Days.get(8)); - Assertions.assertEquals(false, flag90Days.get(9)); + Assertions.assertEquals(1, resultMap.get(1).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(2).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(3).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(4).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(5).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(6).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(7).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(8).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(9).getFlag90days()); } - private void findFlags5000(List fids, Map flag5000Map, Map flag90Days) { - ruPastDuarearDataList.sort(Comparator.comparing(FlagsService.RuPastDuarearData::getFid) // сортируем сначала по фиду - .thenComparing(FlagsService.RuPastDuarearData::getReportedDate, Comparator.reverseOrder())); // Затем сортируем по REPORTED_DT по убыванию - - LocalDate yearAgo = LocalDate.parse("2024-09-09"); - - for (Integer fid : fids) { - Iterator iterator = ruPastDuarearDataList.iterator(); - BigInteger prevAccSerialNum = null; - while (iterator.hasNext()) { - FlagsService.RuPastDuarearData data = iterator.next(); - - if (fid.equals(data.getFid())) { - if (flag5000Map.get(fid) == null) { // Первая запись в списке с макс REPORTED_DT. Нас интересует только она для флага 5000 - if (prevAccSerialNum == null || !prevAccSerialNum.equals(data.getAccSerialNum())) { - if (data.getAmtPastDue() != null && data.getAmtPastDue().compareTo(BigDecimal.valueOf(5000)) >= 0) { - flag5000Map.put(fid, true); - } else { - flag5000Map.put(fid, false); - } - prevAccSerialNum = data.getAccSerialNum(); - } - } - - if (flag90Days.get(fid) != null && flag90Days.get(fid).equals(Boolean.TRUE)) { - iterator.remove(); - break; - } - - if (flag90Days.get(fid) == null || flag90Days.get(fid).equals(Boolean.FALSE)) { - if (data.getDaysPastDue() != null) { - boolean has90daysFlag = data.getDaysPastDue().compareTo(BigInteger.valueOf(90)) >= 0 && !data.getReportedDate().isBefore(yearAgo); - if (has90daysFlag) { - flag90Days.put(fid, true); - iterator.remove(); - break; - } else { - flag90Days.put(fid, false); - iterator.remove(); - } - } else { - iterator.remove(); - } - } - } - } - if (flag5000Map.get(fid) == null) { - flag5000Map.put(fid, false); - } - if (flag90Days.get(fid) == null) { - flag90Days.put(fid, false); - } - } - } private final List ruPastDuarearDataList = new ArrayList<>( List.of( - new FlagsService.RuPastDuarearData(1, BigInteger.valueOf(1), LocalDate.parse("2025-12-31"), BigDecimal.valueOf(36000.00), BigInteger.valueOf(159)), // просрочка 5000 и 90_days по этой записи + new FlagsService.RuPastDuarearData(1, BigInteger.valueOf(1), calcLocalDateFromInitialDate("2025-05-31"), BigDecimal.valueOf(36000.00), BigInteger.valueOf(159)), // просрочка 5000 и 90_days по этой записи - new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2021-12-29"), BigDecimal.valueOf(3200.00), null), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2022-01-28"), BigDecimal.valueOf(6700.00), null), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2025-04-06"), BigDecimal.valueOf(5700.00), null), // просрочка 5000 по этой записи + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), calcLocalDateFromInitialDate("2021-12-29"), BigDecimal.valueOf(3200.00), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), calcLocalDateFromInitialDate("2022-01-28"), BigDecimal.valueOf(6700.00), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), calcLocalDateFromInitialDate("2025-04-06"), BigDecimal.valueOf(5700.00), null), // просрочка 5000 по этой записи - new FlagsService.RuPastDuarearData(3, BigInteger.valueOf(3), LocalDate.parse("2025-01-14"), BigDecimal.valueOf(300000000000.00), null), + new FlagsService.RuPastDuarearData(3, BigInteger.valueOf(3), calcLocalDateFromInitialDate("2025-01-14"), BigDecimal.valueOf(300000000000.00), null), - new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), LocalDate.parse("2021-11-29"), BigDecimal.valueOf(0.00), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(2, BigInteger.valueOf(2), calcLocalDateFromInitialDate("2021-11-29"), BigDecimal.valueOf(0.00), null), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(4, BigInteger.valueOf(4), LocalDate.parse("2025-12-06"), BigDecimal.valueOf(4000.00), null), // нет просрочек + new FlagsService.RuPastDuarearData(4, BigInteger.valueOf(4), calcLocalDateFromInitialDate("2025-12-06"), BigDecimal.valueOf(4000.00), null), // нет просрочек - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-04-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(182)), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-05-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(212)), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-05-31"), BigDecimal.valueOf(2000.31), null), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-07-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(273)), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-07-31"), BigDecimal.valueOf(2000.31), null), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-08-04"), BigDecimal.valueOf(7000.31), BigInteger.valueOf(307)), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2025-09-08"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(338)), // не макс reported_date для того же acc_serial_num, просрочка 90_days будет взята у этой записи - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(51), LocalDate.parse("2025-09-09"), BigDecimal.valueOf(7000.31), BigInteger.valueOf(368)), // Просрочка 5000 будет взята у этой записи. Уникальный acc_serial_num с макс reported_dt - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2021-11-04"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(399)), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), LocalDate.parse("2022-01-04"), BigDecimal.valueOf(5000.31), BigInteger.valueOf(5)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2021-04-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(182)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2021-05-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(212)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2021-05-31"), BigDecimal.valueOf(2000.31), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2021-07-01"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(273)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2021-07-31"), BigDecimal.valueOf(2000.31), null), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2021-08-04"), BigDecimal.valueOf(7000.31), BigInteger.valueOf(307)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2025-09-08"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(338)), // не макс reported_date для того же acc_serial_num, просрочка 90_days будет взята у этой записи + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(51), calcLocalDateFromInitialDate("2025-09-09"), BigDecimal.valueOf(7000.31), BigInteger.valueOf(368)), // Просрочка 5000 будет взята у этой записи. Уникальный acc_serial_num с макс reported_dt + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2021-11-04"), BigDecimal.valueOf(2000.31), BigInteger.valueOf(399)), // не макс reported_date для того же acc_serial_num + new FlagsService.RuPastDuarearData(5, BigInteger.valueOf(5), calcLocalDateFromInitialDate("2022-01-04"), BigDecimal.valueOf(5000.31), BigInteger.valueOf(5)), // не макс reported_date для того же acc_serial_num - new FlagsService.RuPastDuarearData(6, BigInteger.valueOf(6), LocalDate.parse("2025-09-09"), BigDecimal.valueOf(3000), null), // Нет просрочки 5000, нет просрочки 90_days (90_days будет в RU_PAYMT) + new FlagsService.RuPastDuarearData(6, BigInteger.valueOf(6), calcLocalDateFromInitialDate("2025-09-09"), BigDecimal.valueOf(3000), null), // Нет просрочки 5000, нет просрочки 90_days (90_days будет в RU_PAYMT) - new FlagsService.RuPastDuarearData(7, BigInteger.valueOf(7), LocalDate.parse("2025-09-09"), BigDecimal.valueOf(5500), BigInteger.valueOf(90)), // не макс reported_date для того же acc_serial_num. 90_days будет взята отсюда - new FlagsService.RuPastDuarearData(7, BigInteger.valueOf(7), LocalDate.parse("2025-09-10"), BigDecimal.valueOf(3000), null) // Нет просрочки 5000. Уникальный acc_serial_num с макс reported_dt + new FlagsService.RuPastDuarearData(7, BigInteger.valueOf(7), calcLocalDateFromInitialDate("2025-09-09"), BigDecimal.valueOf(5500), BigInteger.valueOf(90)), // не макс reported_date для того же acc_serial_num. 90_days будет взята отсюда + new FlagsService.RuPastDuarearData(7, BigInteger.valueOf(7), calcLocalDateFromInitialDate("2025-09-10"), BigDecimal.valueOf(3000), null) // Нет просрочки 5000. Уникальный acc_serial_num с макс reported_dt ) ); + + private LocalDate calcLocalDateFromInitialDate(String date) { + LocalDateTime now = LocalDateTime.now(); + long daysPassed = ChronoUnit.DAYS.between(INITIAL_DATE, now); + return LocalDate.parse(date).plusDays(daysPassed); + } } From d1064b6b9fc5897adc3421e5908493c0aa2326b7 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Thu, 25 Sep 2025 16:51:19 +0300 Subject: [PATCH 13/20] Minor fix calc flag5000 --- .../CreditTrackerApplication.java | 2 ++ .../service/indic/FlagsService.java | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java index fa2c9f9..3dc981c 100644 --- a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java +++ b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java @@ -30,10 +30,12 @@ public class CreditTrackerApplication { for (Integer fid : fids) { FlagsService.FlagsData oldValue = oldMap.get(fid); FlagsService.FlagsData newValue = newMap.get(fid); + System.out.println("FID: " + fid); System.out.println("OLD: " + oldValue.getFlagDebt5000() +"|" +oldValue.getFlag90days()); System.out.println("NEW: " + newValue.getFlagDebt5000() +"|" +newValue.getFlag90days()); System.out.println("_________________________________"); } + System.exit(0); // UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class); // MonitoringService monitoringService = ctx.getBean(MonitoringService.class); // UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class); diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 209cce3..4157c93 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -89,6 +89,12 @@ public class FlagsService { return result; } + /** + * Выбирает дату из RU_PASTDUEARREAR и ищет по ним флаги просрочек + * @param fids - отсортирован по возрастанию фидов + * @param flag5000Map - рещультирующая мапа флагов просрочек debt5000 + * @param flag90Days - рещультирующая мапа флагов просрочек 90_days + */ private void findFlagsRuPastDueArrear(List fids, Map flag5000Map, Map flag90Days) { log.debug("Start calculating flag 5000 and 90 days by RU_PASTDUEARREAR"); List ruPastDuarearDataList = mapper.findRuPastDuarearData(fids); @@ -108,19 +114,18 @@ public class FlagsService { if (prevAccSerialNum == null || !prevAccSerialNum.equals(data.getAccSerialNum())) { if (data.getAmtPastDue() != null && data.getAmtPastDue().compareTo(BigDecimal.valueOf(5000)) >= 0) { flag5000Map.put(fid, 1); - } else { - flag5000Map.put(fid, 0); } prevAccSerialNum = data.getAccSerialNum(); } } - if (flag90Days.get(fid) != null && flag90Days.get(fid) == 1) { + Integer flag90 = flag90Days.get(fid); + if (flag90 != null && flag90 == 1) { iterator.remove(); break; } - if (flag90Days.get(fid) == null || flag90Days.get(fid) == 0) { + if (flag90 == null || flag90 == 0) { if (data.getDaysPastDue() != null) { boolean has90daysFlag = data.getDaysPastDue().compareTo(BigInteger.valueOf(90)) >= 0 && !data.getReportedDate().isBefore(yearAgo); if (has90daysFlag) { @@ -137,12 +142,9 @@ public class FlagsService { } } } - if (flag5000Map.get(fid) == null) { - flag5000Map.put(fid, 0); - } - if (flag90Days.get(fid) == null) { - flag90Days.put(fid, 0); - } + // если мы так и не нашли флаги для этих фидов - выставляем их в 0 + flag5000Map.putIfAbsent(fid, 0); + flag90Days.putIfAbsent(fid, 0); } log.debug("Complete calculating flag 5000 and 90 days by RU_PASTDUEARREAR"); } From 45ca5f4440160697bd0be85eb0f3bba768f475a9 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Thu, 25 Sep 2025 18:02:05 +0300 Subject: [PATCH 14/20] Changed logic for loading flag90days from Ru_Paymt; Added tests --- .../CreditTrackerApplication.java | 27 ---- .../mapper/indic/FlagsMapper.java | 8 +- .../service/indic/FlagsService.java | 132 +++++++----------- .../mybatis/mapper/indic/FlagsMapper.xml | 87 ++---------- .../service/FlagsServiceTest.java | 112 ++++++++++++++- 5 files changed, 177 insertions(+), 189 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java index 3dc981c..891ec93 100644 --- a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java +++ b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java @@ -3,39 +3,12 @@ package ru.nbch.credit_tracker; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; -import ru.nbch.credit_tracker.service.indic.FlagsService; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; @SpringBootApplication public class CreditTrackerApplication { public static void main(String[] args) { ConfigurableApplicationContext ctx = SpringApplication.run(CreditTrackerApplication.class, args); - List fids = new ArrayList<>(); - fids.add(11085548); - fids.add(16442833); - fids.add(109128766); - fids.add(115150769); - fids.add(16442929); - fids.add(114429070); - fids.add(108696892); - FlagsService flagService = ctx.getBean(FlagsService.class); - Map oldMap = flagService.findFlags(fids); - Map newMap = flagService.findFlags2(fids); - System.out.println(oldMap); - System.out.println(newMap); - for (Integer fid : fids) { - FlagsService.FlagsData oldValue = oldMap.get(fid); - FlagsService.FlagsData newValue = newMap.get(fid); - System.out.println("FID: " + fid); - System.out.println("OLD: " + oldValue.getFlagDebt5000() +"|" +oldValue.getFlag90days()); - System.out.println("NEW: " + newValue.getFlagDebt5000() +"|" +newValue.getFlag90days()); - System.out.println("_________________________________"); - } - System.exit(0); // UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class); // MonitoringService monitoringService = ctx.getBean(MonitoringService.class); // UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class); diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java index 1aa12be..a8c768b 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java @@ -10,12 +10,8 @@ import java.util.List; @Mapper public interface FlagsMapper { - List findFlagsDebt5000(@Param("fids") List fids); - - List findFlag90ByPastDueArrear(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); - - List findFlag90ByRuPaymt(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); - List findRuPastDuarearData(@Param("fids") List fids); + List findRuPaymtData(@Param("fids") List fids, @Param("reportedDate") LocalDate reportedDate); + } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 4157c93..12a2134 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -25,36 +25,6 @@ public class FlagsService { } public Map findFlags(List fids) { - if (fids.isEmpty()) { - return new HashMap<>(); - } - List distinctFids = fids.stream().distinct().toList(); - log.debug("Fids count: {}", distinctFids.size()); - Map flags5000 = calcFlag5000(distinctFids); - Map flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(distinctFids); - Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); - - Map result = new HashMap<>(); - fids.forEach(fid -> { - Integer flagDebt5000 = flags5000.get(fid); - if (flagDebt5000 == null) { - flagDebt5000 = 0; - } - - Integer flag90days = flags90DaysByPastDueArrear.get(fid); - if (flag90days == null || flag90days == 0) { - flag90days = flags90DaysByRuPaymt.get(fid); - if (flag90days == null) { - flag90days = 0; - } - } - result.put(fid, new FlagsData(flagDebt5000, flag90days)); - }); - - return result; - } - - public Map findFlags2(List fids) { if (fids.isEmpty()) { return new HashMap<>(); } @@ -66,8 +36,9 @@ public class FlagsService { log.debug("Fids count: {}", distinctFids.size()); Map flags5000 = new HashMap<>(); Map flags90DaysByPastDueArrear = new HashMap<>(); + Map flags90DaysByRuPaymt = new HashMap<>(); findFlagsRuPastDueArrear(distinctFids, flags5000, flags90DaysByPastDueArrear); - Map flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(distinctFids); + findFlagsRuPaymt(distinctFids, flags90DaysByRuPaymt); Map result = new HashMap<>(); fids.forEach(fid -> { @@ -90,10 +61,17 @@ public class FlagsService { } /** - * Выбирает дату из RU_PASTDUEARREAR и ищет по ним флаги просрочек - * @param fids - отсортирован по возрастанию фидов + * Выбирает данные из RU_PASTDUEARREAR и ищет по ним флаги просрочек + * По FID из RU_PASTDUEARREAR выбираются все уникальные ACC_SERIAL_NUM с максимальной датой REPORTED_DT. + * Если по хотя бы одной такой записи AMT_PAST_DUE > 5000, то flag500 устанавливается в 1. + * Если ни по одному счёту условие не выполняется или счетов вообще не найдено, флаг = 0. + * + * flag90Days = 1, если выполняется хотя бы одно из условий: + * В INDIC.RU_PASTDUEARREAR у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. + * Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0. + * @param fids - отсортирован по возрастанию фидов * @param flag5000Map - рещультирующая мапа флагов просрочек debt5000 - * @param flag90Days - рещультирующая мапа флагов просрочек 90_days + * @param flag90Days - рещультирующая мапа флагов просрочек 90_days */ private void findFlagsRuPastDueArrear(List fids, Map flag5000Map, Map flag90Days) { log.debug("Start calculating flag 5000 and 90 days by RU_PASTDUEARREAR"); @@ -150,50 +128,49 @@ public class FlagsService { } /** - * По FID из RU_PASTDUEARREAR выбираются все уникальные ACC_SERIAL_NUM с максимальной датой REPORTED_DT. - * Если по хотя бы одной такой записи AMT_PAST_DUE > 5000, то флаг устанавливается в 1. - * Если ни по одному счёту условие не выполняется или счетов вообще не найдено, флаг = 0. - */ - private Map calcFlag5000(List fids) { - log.debug("Start calculating flag 5000"); - List rows = mapper.findFlagsDebt5000(fids); - Map res = makeFidFlagMapByRs(fids, rows); - log.debug("Complete calculating flag 5000"); - return res; - } - - /** - * Флаг = 1, если выполняется хотя бы одно из условий: - * В INDIC.RU_PASTDUEARREAR или RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. + * Выбирает данные из RU_PAYMT и ищет по ним флаги просрочек 90Days + * + * flag90Days = 1, если выполняется хотя бы одно из условий: + * В INDIC.RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. * Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0. + * @param fids - отсортирован по возрастанию фидов + * @param flag90Days - рещультирующая мапа флагов просрочек 90_days */ - private Map calcFlag90DaysByPastDueArrear(List fids) { - log.debug("Start calculating flag 90 by PastDueArrear"); + private void findFlagsRuPaymt(List fids, Map flag90Days) { LocalDate yearAgo = LocalDate.now().minusYears(1); - List rows = mapper.findFlag90ByPastDueArrear(fids, yearAgo); - Map res = makeFidFlagMapByRs(fids, rows); - log.debug("Complete calculating flag 90 by PastDueArrear"); - return res; - } + List ruPaymtDataList = mapper.findRuPaymtData(fids, yearAgo); + ruPaymtDataList.sort(Comparator.comparing(RuPaymtData::getFid, Comparator.naturalOrder())); - private Map calcFlag90DaysByRuPaymt(List fids) { - log.debug("Start calculating flag 90 by ruPaymt"); - LocalDate yearAgo = LocalDate.now().minusYears(1); - List rows = mapper.findFlag90ByRuPaymt(fids, yearAgo); - Map res = makeFidFlagMapByRs(fids, rows); - log.debug("Complete calculating flag 90 by ruPaymt"); - return res; - } - - private Map makeFidFlagMapByRs(List fids, List rows) { - Map result = new HashMap<>(); for (Integer fid : fids) { - result.put(fid, 0); + Iterator iterator = ruPaymtDataList.iterator(); + while (iterator.hasNext()) { + RuPaymtData data = iterator.next(); + if (fid.equals(data.getFid())) { + Integer flag90 = flag90Days.get(fid); + if (flag90 != null && flag90 == 1) { + iterator.remove(); + break; + } + + if (flag90 == null || flag90 == 0) { + if (data.getDaysPastDue() != null) { + boolean has90daysFlag = data.getDaysPastDue().compareTo(BigInteger.valueOf(90)) >= 0 && !data.getReportedDate().isBefore(yearAgo); + if (has90daysFlag) { + flag90Days.put(fid, 1); + iterator.remove(); + break; + } else { + flag90Days.put(fid, 0); + iterator.remove(); + } + } else { + iterator.remove(); + } + } + } + } + flag90Days.putIfAbsent(fid, 0); } - for (FlagFidData row : rows) { - result.put(row.getFid(), row.getFlag() != null ? row.getFlag() : 0); - } - return result; } @Getter @@ -204,13 +181,6 @@ public class FlagsService { private Integer flag90days; } - @Getter - @Setter - public static class FlagFidData { - private Integer fid; - private Integer flag; - } - @Getter @Setter @AllArgsConstructor @@ -225,7 +195,9 @@ public class FlagsService { @Getter @Setter - public static class Flag90DaysData { + @AllArgsConstructor + @NoArgsConstructor + public static class RuPaymtData { private Integer fid; private BigInteger daysPastDue; private LocalDate reportedDate; diff --git a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml index ab3bb98..a186744 100644 --- a/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml +++ b/src/main/resources/mybatis/mapper/indic/FlagsMapper.xml @@ -5,11 +5,6 @@ - - - - - @@ -18,6 +13,12 @@ + + + + + + - - - - - + SELECT FID, DAYS_PAST_DUE, REPORTED_DT + FROM RU_PAYMT + WHERE FID IN + + #{fid} + + AND REPORTED_DT >= #{reportedDate} + with UR \ No newline at end of file diff --git a/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java b/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java index 3191a7c..2284db9 100644 --- a/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java +++ b/src/test/java/ru/nbch/credit_tracker/service/FlagsServiceTest.java @@ -29,12 +29,11 @@ public class FlagsServiceTest { FlagsMapper flagsMapper; @Test - public void testfindFlagsWithoutRuPaymt() { - + public void testfindFlags() { FlagsService flagsService = new FlagsService(flagsMapper); when(flagsMapper.findRuPastDuarearData(Mockito.anyList())).thenReturn(ruPastDuarearDataList); - when(flagsMapper.findFlag90ByRuPaymt(Mockito.anyList(), Mockito.any())).thenReturn(new ArrayList<>()); + when(flagsMapper.findRuPaymtData(Mockito.anyList(), Mockito.any())).thenReturn(ruPaymtDataList); List fids = new ArrayList<>(List.of( 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR @@ -48,7 +47,50 @@ public class FlagsServiceTest { 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT ); - Map resultMap = flagsService.findFlags2(fids); + Map resultMap = flagsService.findFlags(fids); + + Assertions.assertEquals(9, resultMap.size()); + Assertions.assertEquals(1, resultMap.get(1).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(2).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(3).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(4).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(5).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(6).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(7).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(8).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(9).getFlagDebt5000()); + + Assertions.assertEquals(1, resultMap.get(1).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(2).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(3).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(4).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(5).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(6).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(7).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(8).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(9).getFlag90days()); + } + + @Test + public void testfindFlagsWithoutRuPaymt() { + FlagsService flagsService = new FlagsService(flagsMapper); + + when(flagsMapper.findRuPastDuarearData(Mockito.anyList())).thenReturn(ruPastDuarearDataList); + when(flagsMapper.findRuPaymtData(Mockito.anyList(), Mockito.any())).thenReturn(new ArrayList<>()); + + List fids = new ArrayList<>(List.of( + 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 2, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Нет просрочки 90_days в RU_PAYMT + 3, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 4, // нет никаких просрочек + 7, // нет просрочки 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 5, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 6, // нет просрочки 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 8, // нету в RU_PASTDUEARREAR, есть в RU_PAYMT + 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT + ); + + Map resultMap = flagsService.findFlags(fids); Assertions.assertEquals(9, resultMap.size()); Assertions.assertEquals(1, resultMap.get(1).getFlagDebt5000()); @@ -73,6 +115,50 @@ public class FlagsServiceTest { } + @Test + public void testfindFlagsWithoutRuPastduearreae() { + FlagsService flagsService = new FlagsService(flagsMapper); + + when(flagsMapper.findRuPastDuarearData(Mockito.anyList())).thenReturn(new ArrayList<>()); + when(flagsMapper.findRuPaymtData(Mockito.anyList(), Mockito.any())).thenReturn(ruPaymtDataList); + + List fids = new ArrayList<>(List.of( + 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 2, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Нет просрочки 90_days в RU_PAYMT + 3, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 4, // нет никаких просрочек + 7, // нет просрочки 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 5, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 6, // нет просрочки 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 8, // нету в RU_PASTDUEARREAR, есть в RU_PAYMT + 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT + ); + + Map resultMap = flagsService.findFlags(fids); + + Assertions.assertEquals(9, resultMap.size()); + Assertions.assertEquals(0, resultMap.get(1).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(2).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(3).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(4).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(5).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(6).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(7).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(8).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(9).getFlagDebt5000()); + + Assertions.assertEquals(0, resultMap.get(1).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(2).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(3).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(4).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(5).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(6).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(7).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(8).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(9).getFlag90days()); + } + + private final List ruPastDuarearDataList = new ArrayList<>( List.of( new FlagsService.RuPastDuarearData(1, BigInteger.valueOf(1), calcLocalDateFromInitialDate("2025-05-31"), BigDecimal.valueOf(36000.00), BigInteger.valueOf(159)), // просрочка 5000 и 90_days по этой записи @@ -106,6 +192,24 @@ public class FlagsServiceTest { ) ); + private final List ruPaymtDataList = new ArrayList<>( + List.of( + new FlagsService.RuPaymtData(1, null, calcLocalDateFromInitialDate("2024-11-09")), + new FlagsService.RuPaymtData(2, BigInteger.valueOf(30), calcLocalDateFromInitialDate("2024-11-09")), + new FlagsService.RuPaymtData(3, null, calcLocalDateFromInitialDate("2024-11-10")), + new FlagsService.RuPaymtData(3, BigInteger.valueOf(100), calcLocalDateFromInitialDate("2024-11-15")), + new FlagsService.RuPaymtData(4, BigInteger.valueOf(10), calcLocalDateFromInitialDate("2024-11-15")), + new FlagsService.RuPaymtData(5, BigInteger.valueOf(10), calcLocalDateFromInitialDate("2023-11-15")), + new FlagsService.RuPaymtData(6, BigInteger.valueOf(90), calcLocalDateFromInitialDate("2024-11-09")), + new FlagsService.RuPaymtData(6, BigInteger.valueOf(80), calcLocalDateFromInitialDate("2024-10-09")), + new FlagsService.RuPaymtData(7, null, calcLocalDateFromInitialDate("2024-10-09")), + new FlagsService.RuPaymtData(8, null, calcLocalDateFromInitialDate("2024-10-09")), + new FlagsService.RuPaymtData(8, BigInteger.valueOf(83), calcLocalDateFromInitialDate("2024-10-09")), + new FlagsService.RuPaymtData(8, BigInteger.valueOf(91), calcLocalDateFromInitialDate("2024-10-09")), + new FlagsService.RuPaymtData(9, BigInteger.valueOf(100), calcLocalDateFromInitialDate("2024-10-09")) + ) + ); + private LocalDate calcLocalDateFromInitialDate(String date) { LocalDateTime now = LocalDateTime.now(); long daysPassed = ChronoUnit.DAYS.between(INITIAL_DATE, now); From e70f07b04bec37981af38b6cee7361a141f6a83e Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Thu, 25 Sep 2025 18:03:07 +0300 Subject: [PATCH 15/20] Added debug logs --- .../java/ru/nbch/credit_tracker/service/indic/FlagsService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java index 12a2134..09fbac6 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java @@ -137,6 +137,7 @@ public class FlagsService { * @param flag90Days - рещультирующая мапа флагов просрочек 90_days */ private void findFlagsRuPaymt(List fids, Map flag90Days) { + log.debug("Start calculating flag 90 days by RU_PAPAYMT"); LocalDate yearAgo = LocalDate.now().minusYears(1); List ruPaymtDataList = mapper.findRuPaymtData(fids, yearAgo); ruPaymtDataList.sort(Comparator.comparing(RuPaymtData::getFid, Comparator.naturalOrder())); @@ -171,6 +172,7 @@ public class FlagsService { } flag90Days.putIfAbsent(fid, 0); } + log.debug("Complete calculating flag 90 days by RU_PAPAYMT"); } @Getter From 4f55c0df4258f68b2bbc2ddd21360f9953c0e797 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Mon, 29 Sep 2025 10:10:21 +0300 Subject: [PATCH 16/20] Added flags fetching by jdbc template --- .../datasource/IndicDataSourceConfig.java | 6 + .../service/indic/FlagsService2.java | 256 ++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java diff --git a/src/main/java/ru/nbch/credit_tracker/config/datasource/IndicDataSourceConfig.java b/src/main/java/ru/nbch/credit_tracker/config/datasource/IndicDataSourceConfig.java index e83429b..b802514 100644 --- a/src/main/java/ru/nbch/credit_tracker/config/datasource/IndicDataSourceConfig.java +++ b/src/main/java/ru/nbch/credit_tracker/config/datasource/IndicDataSourceConfig.java @@ -9,6 +9,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.Resource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; import javax.sql.DataSource; @@ -47,5 +48,10 @@ public class IndicDataSourceConfig extends DatabaseConfig { sqlSessionFactoryBean.setTypeHandlersPackage("ru.nbch.credit_tracker.config.datasource.handlers"); return sqlSessionFactoryBean.getObject(); } + + @Bean(name = "jdbcTemplateIndic") + public NamedParameterJdbcTemplate getNamedParameterJdbcTemplate(@Qualifier("indicDataSource") DataSource dataSource) { + return new NamedParameterJdbcTemplate(dataSource); + } } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java new file mode 100644 index 0000000..720f8e3 --- /dev/null +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java @@ -0,0 +1,256 @@ +package ru.nbch.credit_tracker.service.indic; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Service +public class FlagsService2 { + + private static final String SELECT_RU_PASDUEARREAR_DATA = + "SELECT FID, ACC_SERIAL_NUM, REPORTED_DT, AMT_PAST_DUE, DAYS_PAST_DUE " + + "FROM RU_PASTDUEARREAR " + + "WHERE FID IN (:fids) " + + "ORDER BY FID, REPORTED_DT DESC " + + "WITH UR"; + + private static final String SELECT_RU_PAYMT_DATA = + "SELECT FID, DAYS_PAST_DUE, REPORTED_DT " + + "FROM RU_PAYMT " + + "WHERE FID IN (:fids) " + + "AND REPORTED_DT >= :reportedDate " + + "WITH UR"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + + public FlagsService2(@Qualifier("jdbcTemplateIndic") NamedParameterJdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + public Map findFlags(List fids) { + if (fids.isEmpty()) { + return new HashMap<>(); + } + List distinctFids = fids.stream() + .distinct() + .sorted(Comparator.naturalOrder()) + .toList(); + + log.debug("Fids count: {}", distinctFids.size()); + Map flags5000 = new HashMap<>(); + Map flags90DaysByPastDueArrear = new HashMap<>(); + Map flags90DaysByRuPaymt = new HashMap<>(); + findFlagsRuPastDueArrear(distinctFids, flags5000, flags90DaysByPastDueArrear); + findFlagsRuPaymt(distinctFids, flags90DaysByRuPaymt); + + Map result = new HashMap<>(); + fids.forEach(fid -> { + Integer flagDebt5000 = flags5000.get(fid); + if (flagDebt5000 == null) { + flagDebt5000 = 0; + } + + Integer flag90days = flags90DaysByPastDueArrear.get(fid); + if (flag90days == null || flag90days == 0) { + flag90days = flags90DaysByRuPaymt.get(fid); + if (flag90days == null) { + flag90days = 0; + } + } + result.put(fid, new FlagsService.FlagsData(flagDebt5000, flag90days)); + }); + + return result; + } + + /** + * Выбирает данные из RU_PASTDUEARREAR и ищет по ним флаги просрочек + * По FID из RU_PASTDUEARREAR выбираются все уникальные ACC_SERIAL_NUM с максимальной датой REPORTED_DT. + * Если по хотя бы одной такой записи AMT_PAST_DUE > 5000, то flag500 устанавливается в 1. + * Если ни по одному счёту условие не выполняется или счетов вообще не найдено, флаг = 0. + *

+ * flag90Days = 1, если выполняется хотя бы одно из условий: + * В INDIC.RU_PASTDUEARREAR у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. + * Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0. + * + * @param fids - отсортирован по возрастанию фидов + * @param flag5000Map - рещультирующая мапа флагов просрочек debt5000 + * @param flag90Days - рещультирующая мапа флагов просрочек 90_days + */ + private void findFlagsRuPastDueArrear(List fids, Map flag5000Map, Map flag90Days) { + log.debug("Start calculating flag 5000 and 90 days by RU_PASTDUEARREAR 2"); + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("fids", fids); + + jdbcTemplate.query(SELECT_RU_PASDUEARREAR_DATA, params, (ResultSet rs) -> { + Integer currentFid = null; + BigInteger prevAccSerialNum = null; + boolean processed5000ForCurrentId = false; + boolean processed90DaysForCurrentId = false; + LocalDate yearAgo = LocalDate.now().minusYears(1); + + while (rs.next()) { + Integer fid = rs.getInt("FID"); + + // Переход к новому fid. Сброс состояния + if (!fid.equals(currentFid)) { + currentFid = fid; + prevAccSerialNum = null; + processed5000ForCurrentId = false; + processed90DaysForCurrentId = false; + + // Инициализируем значения по умолчанию + flag5000Map.putIfAbsent(fid, 0); + flag90Days.putIfAbsent(fid, 0); + } + + // Если для текущего FID уже обработали оба флага, пропускаем остальные записи + if (processed5000ForCurrentId && processed90DaysForCurrentId) { + continue; + } + + // Обработка флага 5000 + if (!processed5000ForCurrentId && flag5000Map.get(fid) == 0) { + BigInteger currentAccSerialNum = getBigInteger(rs, "ACC_SERIAL_NUM"); + BigDecimal amtPastDue = getBigDecimal(rs, "AMT_PAST_DUE"); + + if (prevAccSerialNum == null || !prevAccSerialNum.equals(currentAccSerialNum)) { + if (amtPastDue != null && amtPastDue.compareTo(BigDecimal.valueOf(5000)) >= 0) { + flag5000Map.put(fid, 1); + processed5000ForCurrentId = true; + } + prevAccSerialNum = currentAccSerialNum; + } + } + + // Обработка флага 90 дней + if (!processed90DaysForCurrentId) { + Integer currentFlag90 = flag90Days.get(fid); + + if (currentFlag90 != null && currentFlag90 == 1) { + processed90DaysForCurrentId = true; + continue; + } + + if (currentFlag90 == null || currentFlag90 == 0) { + BigInteger days = getBigInteger(rs, "DAYS_PAST_DUE"); + LocalDate reportedDate = rs.getObject("REPORTED_DT", LocalDate.class); + + if (days != null) { + boolean has90daysFlag = days.compareTo(BigInteger.valueOf(90)) >= 0 + && reportedDate != null && !reportedDate.isBefore(yearAgo); + + if (has90daysFlag) { + flag90Days.put(fid, 1); + processed90DaysForCurrentId = true; + } else { + flag90Days.put(fid, 0); + } + } else { + flag90Days.put(fid, 0); + } + } + } + } + return null; + }); + log.debug("Complete calculating flag 5000 and 90 days by RU_PASTDUEARREAR 2"); + } + + /** + * Выбирает данные из RU_PAYMT и ищет по ним флаги просрочек 90Days + *

+ * flag90Days = 1, если выполняется хотя бы одно из условий: + * В INDIC.RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев. + * Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0. + * + * @param fids - отсортирован по возрастанию фидов + * @param flag90Days - рещультирующая мапа флагов просрочек 90_days + */ + private void findFlagsRuPaymt(List fids, Map flag90Days) { + log.debug("Start calculating flag 90 days by RU_PAPAYMT 2"); + LocalDate yearAgo = LocalDate.now().minusYears(1); + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("fids", fids); + params.addValue("reportedDate", yearAgo); + jdbcTemplate.query(SELECT_RU_PAYMT_DATA, params, rs -> { + Integer currentFid = null; + boolean processed90DaysForCurrentId = false; + + while (rs.next()) { + Integer fid = rs.getInt("FID"); + + // Переход к новому fid. Сброс состояния + if (!fid.equals(currentFid)) { + currentFid = fid; + processed90DaysForCurrentId = false; + + // Инициализируем значения по умолчанию + flag90Days.putIfAbsent(fid, 0); + } + + // Обработка флага 90 дней + if (!processed90DaysForCurrentId) { + Integer currentFlag90 = flag90Days.get(fid); + + if (currentFlag90 != null && currentFlag90 == 1) { + processed90DaysForCurrentId = true; + continue; + } + + if (currentFlag90 == null || currentFlag90 == 0) { + BigInteger days = getBigInteger(rs, "DAYS_PAST_DUE"); + LocalDate date = rs.getObject("REPORTED_DT", LocalDate.class); + + if (days != null) { + boolean has90daysFlag = days.compareTo(BigInteger.valueOf(90)) >= 0 + && date != null && !date.isBefore(yearAgo); + + if (has90daysFlag) { + flag90Days.put(fid, 1); + processed90DaysForCurrentId = true; + } else { + flag90Days.put(fid, 0); + } + } else { + flag90Days.put(fid, 0); + } + } + } + } + return null; + }); + log.debug("Complete calculating flag 90 days by RU_PAPAYMT 2"); + } + + + private BigInteger getBigInteger(ResultSet rs, String columnName) { + try { + String value = rs.getString(columnName); + return value != null ? new BigInteger(value) : null; + } catch (SQLException e) { + return null; + } + } + + private BigDecimal getBigDecimal(ResultSet rs, String columnName) { + try { + return rs.getBigDecimal(columnName); + } catch (SQLException e) { + return null; + } + } +} From 8c8ee75292af2befbb8b0b4deff4f4e595ea4329 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Mon, 29 Sep 2025 11:51:46 +0300 Subject: [PATCH 17/20] Added test for FlagsService2 --- pom.xml | 5 + .../service/FlagsService2Test.java | 280 ++++++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 src/test/java/ru/nbch/credit_tracker/service/FlagsService2Test.java diff --git a/pom.xml b/pom.xml index ffd17aa..c57d7bc 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,11 @@ 3.0.4 test + + com.h2database + h2 + test + org.mockito mockito-junit-jupiter diff --git a/src/test/java/ru/nbch/credit_tracker/service/FlagsService2Test.java b/src/test/java/ru/nbch/credit_tracker/service/FlagsService2Test.java new file mode 100644 index 0000000..77bce91 --- /dev/null +++ b/src/test/java/ru/nbch/credit_tracker/service/FlagsService2Test.java @@ -0,0 +1,280 @@ +package ru.nbch.credit_tracker.service; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import ru.nbch.credit_tracker.service.indic.FlagsService; +import ru.nbch.credit_tracker.service.indic.FlagsService2; + +import javax.sql.DataSource; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class FlagsService2Test { + + private static final LocalDate INITIAL_DATE = LocalDate.parse("2025-09-25"); + + private static final String RU_PASTDUEAREAR_CREATE = """ + create table RU_PASTDUEARREAR + ( + FID INTEGER not null, + SERIAL_NUM BIGINT not null primary key, + ACC_SERIAL_NUM BIGINT not null, + REPORTED_DT DATE not null, + AMT_PAST_DUE DECIMAL(17, 2), + DAYS_PAST_DUE BIGINT + ); + """; + + private static final String RU_PASTDUEAREAR_DROP = """ + DROP TABLE IF EXISTS RU_PASTDUEARREAR; + """; + + private static final String RU_PAYMT_DROP = """ + DROP TABLE IF EXISTS RU_PAYMT; + """; + + private static final String RU_PAYMT_CREATE = """ + create table RU_PAYMT + ( + FID INTEGER not null, + SERIAL_NUM BIGINT not null primary key, + DAYS_PAST_DUE INTEGER, + REPORTED_DT DATE not null + ); + """; + + private static final String FILL_RU_PASTDUEARREAR = String.format(""" + INSERT INTO RU_PASTDUEARREAR VALUES + (1, 1, 1, '%s', 36000.00, 159), -- просрочка 5000 и 90_days по этой записи + (2, 2, 2, '%s', 3200.00, null), -- не макс reported_date для того же acc_serial_num + (2, 3, 2, '%s', 6700.00, null), -- не макс reported_date для того же acc_serial_num + (2, 4, 2, '%s', 5700.00, null), -- просрочка 5000 по этой записи + (3, 5, 3, '%s', 300000000000.00, null), + (2, 6, 2, '%s', 0.00, null), -- не макс reported_date для того же acc_serial_num + (4, 7, 4, '%s', 4000.00, null), -- нет просрочек + (5, 8, 5, '%s', 2000.31, 182), -- не макс reported_date для того же acc_serial_num + (5, 9, 5, '%s', 2000.31, 212), -- не макс reported_date для того же acc_serial_num + (5, 10, 5, '%s', 2000.31, null), -- не макс reported_date для того же acc_serial_num + (5, 11, 5, '%s', 2000.31, 273), -- не макс reported_date для того же acc_serial_num + (5, 12, 5, '%s', 2000.31, null), -- не макс reported_date для того же acc_serial_num + (5, 13, 5, '%s', 7000.31, 307), -- не макс reported_date для того же acc_serial_num + (5, 14, 5, '%s', 2000.31, 338), -- не макс reported_date для того же acc_serial_num, просрочка 90_days будет взята у этой записи + (5, 15, 5, '%s', 7000.31, 368), -- Просрочка 5000 будет взята у этой записи. Уникальный acc_serial_num с макс reported_dt + (5, 16, 5, '%s', 2000.31, 399), -- не макс reported_date для того же acc_serial_num + (5, 17, 5, '%s', 5000.31, 5), -- не макс reported_date для того же acc_serial_num + (6, 18, 6, '%s', 3000, null), -- Нет просрочки 5000, нет просрочки 90_days (90_days будет в RU_PAYMT) + (7, 19, 7, '%s', 5500, 90), -- не макс reported_date для того же acc_serial_num. 90_days будет взята отсюда + (7, 20, 7, '%s', 3000, null) -- Нет просрочки 5000. Уникальный acc_serial_num с макс reported_dt + """, + calcLocalDateFromInitialDate("2025-05-31"), + calcLocalDateFromInitialDate("2021-12-29"), + calcLocalDateFromInitialDate("2022-01-28"), + calcLocalDateFromInitialDate("2025-04-06"), + calcLocalDateFromInitialDate("2025-01-14"), + calcLocalDateFromInitialDate("2021-11-29"), + calcLocalDateFromInitialDate("2025-12-06"), + calcLocalDateFromInitialDate("2021-04-01"), + calcLocalDateFromInitialDate("2021-05-01"), + calcLocalDateFromInitialDate("2021-05-31"), + calcLocalDateFromInitialDate("2021-07-01"), + calcLocalDateFromInitialDate("2021-07-31"), + calcLocalDateFromInitialDate("2021-08-04"), + calcLocalDateFromInitialDate("2025-09-08"), + calcLocalDateFromInitialDate("2025-09-09"), + calcLocalDateFromInitialDate("2021-11-04"), + calcLocalDateFromInitialDate("2022-01-04"), + calcLocalDateFromInitialDate("2025-09-09"), + calcLocalDateFromInitialDate("2025-09-09"), + calcLocalDateFromInitialDate("2025-09-10")); + + private static final String FILL_RU_PAYMT = String.format(""" + INSERT INTO RU_PAYMT VALUES + (1, 1, null, '%s'), + (2, 2, 30, '%s'), + (3, 3, null, '%s'), + (3, 4, 100, '%s'), + (4, 5, 10, '%s'), + (5, 6, 10, '%s'), + (6, 7, 90, '%s'), + (6, 8, 80, '%s'), + (7, 9, null, '%s'), + (8, 10, null, '%s'), + (8, 11, 83, '%s'), + (8, 12, 91, '%s'), + (9, 13, 100, '%s') + """, + calcLocalDateFromInitialDate("2024-11-09"), + calcLocalDateFromInitialDate("2024-11-09"), + calcLocalDateFromInitialDate("2024-11-10"), + calcLocalDateFromInitialDate("2024-11-15"), + calcLocalDateFromInitialDate("2024-11-15"), + calcLocalDateFromInitialDate("2023-11-15"), + calcLocalDateFromInitialDate("2024-11-09"), + calcLocalDateFromInitialDate("2024-10-09"), + calcLocalDateFromInitialDate("2024-10-09"), + calcLocalDateFromInitialDate("2024-10-09"), + calcLocalDateFromInitialDate("2024-10-09"), + calcLocalDateFromInitialDate("2024-10-09"), + calcLocalDateFromInitialDate("2024-10-09")); + + + private DataSource dataSource; + private NamedParameterJdbcTemplate jdbcTemplate; + private FlagsService2 flagsService; + + @BeforeEach + void setUp() { + dataSource = DataSourceBuilder.create() + .url("jdbc:h2:mem:testdb;DB_CLOSE_DELAY=-1;MODE=DB2;DATABASE_TO_LOWER=TRUE") + .driverClassName("org.h2.Driver") + .username("sa") + .password("") + .build(); + + jdbcTemplate = new NamedParameterJdbcTemplate(dataSource); + + jdbcTemplate.getJdbcTemplate().execute(RU_PASTDUEAREAR_DROP); + jdbcTemplate.getJdbcTemplate().execute(RU_PAYMT_DROP); + + jdbcTemplate.getJdbcTemplate().execute(RU_PASTDUEAREAR_CREATE); + jdbcTemplate.getJdbcTemplate().execute(RU_PAYMT_CREATE); + + flagsService = new FlagsService2(jdbcTemplate); + } + + @Test + public void testfindFlags() { + jdbcTemplate.getJdbcTemplate().execute(FILL_RU_PASTDUEARREAR); + jdbcTemplate.getJdbcTemplate().execute(FILL_RU_PAYMT); + + List fids = new ArrayList<>(List.of( + 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 2, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Нет просрочки 90_days в RU_PAYMT + 3, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 4, // нет никаких просрочек + 7, // нет просрочки 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 5, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 6, // нет просрочки 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 8, // нету в RU_PASTDUEARREAR, есть в RU_PAYMT + 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT + ); + + Map resultMap = flagsService.findFlags(fids); + + Assertions.assertEquals(9, resultMap.size()); + Assertions.assertEquals(1, resultMap.get(1).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(2).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(3).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(4).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(5).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(6).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(7).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(8).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(9).getFlagDebt5000()); + + Assertions.assertEquals(1, resultMap.get(1).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(2).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(3).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(4).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(5).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(6).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(7).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(8).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(9).getFlag90days()); + } + + @Test + public void testfindFlagsWithoutRuPaymt() { + jdbcTemplate.getJdbcTemplate().execute(FILL_RU_PASTDUEARREAR); + + List fids = new ArrayList<>(List.of( + 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 2, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Нет просрочки 90_days в RU_PAYMT + 3, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 4, // нет никаких просрочек + 7, // нет просрочки 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 5, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 6, // нет просрочки 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 8, // нету в RU_PASTDUEARREAR, есть в RU_PAYMT + 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT + ); + + Map resultMap = flagsService.findFlags(fids); + + Assertions.assertEquals(9, resultMap.size()); + Assertions.assertEquals(1, resultMap.get(1).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(2).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(3).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(4).getFlagDebt5000()); + Assertions.assertEquals(1, resultMap.get(5).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(6).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(7).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(8).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(9).getFlagDebt5000()); + + Assertions.assertEquals(1, resultMap.get(1).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(2).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(3).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(4).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(5).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(6).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(7).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(8).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(9).getFlag90days()); + } + + @Test + public void testfindFlagsWithoutRuPastduearreae() { + jdbcTemplate.getJdbcTemplate().execute(FILL_RU_PAYMT); + + List fids = new ArrayList<>(List.of( + 1, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 2, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Нет просрочки 90_days в RU_PAYMT + 3, // есть просрочка 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 4, // нет никаких просрочек + 7, // нет просрочки 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 5, // есть просрочка 5000, есть просрочка 90_days в RU_PASTDUEARREAR + 6, // нет просрочки 5000, нет просрочки 90_days в RU_PASTDUEARREAR. Есть просрочка 90_days в RU_PAYMT + 8, // нету в RU_PASTDUEARREAR, есть в RU_PAYMT + 9) // нету в RU_PASTDUEARREAR, нету в RU_PAYMT + ); + + Map resultMap = flagsService.findFlags(fids); + + Assertions.assertEquals(9, resultMap.size()); + Assertions.assertEquals(0, resultMap.get(1).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(2).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(3).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(4).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(5).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(6).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(7).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(8).getFlagDebt5000()); + Assertions.assertEquals(0, resultMap.get(9).getFlagDebt5000()); + + Assertions.assertEquals(0, resultMap.get(1).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(2).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(3).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(4).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(5).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(6).getFlag90days()); + Assertions.assertEquals(0, resultMap.get(7).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(8).getFlag90days()); + Assertions.assertEquals(1, resultMap.get(9).getFlag90days()); + } + + private static LocalDate calcLocalDateFromInitialDate(String date) { + LocalDateTime now = LocalDateTime.now(); + long daysPassed = ChronoUnit.DAYS.between(INITIAL_DATE, now); + return LocalDate.parse(date).plusDays(daysPassed); + } +} + + From e07b2036865b237e0c5d31b8aad8acd5e7ef05aa Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Mon, 29 Sep 2025 13:04:39 +0300 Subject: [PATCH 18/20] Use FlagsService2 --- .../credit_tracker/CreditTrackerApplication.java | 15 +++++++++------ .../credit_tracker/service/SignalService.java | 9 +++------ .../service/indic/FlagsService2.java | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java index 891ec93..d3a78f1 100644 --- a/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java +++ b/src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java @@ -3,18 +3,21 @@ package ru.nbch.credit_tracker; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; +import ru.nbch.credit_tracker.service.MonitoringService; +import ru.nbch.credit_tracker.service.UpdateProcessingPackagesService; +import ru.nbch.credit_tracker.service.UpdatingFlagsService; @SpringBootApplication public class CreditTrackerApplication { public static void main(String[] args) { ConfigurableApplicationContext ctx = SpringApplication.run(CreditTrackerApplication.class, args); -// UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class); -// MonitoringService monitoringService = ctx.getBean(MonitoringService.class); -// UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class); -// updateProcessingPackagesService.run(); -// monitoringService.run(); -// updatingFlagsService.run(); + UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class); + MonitoringService monitoringService = ctx.getBean(MonitoringService.class); + UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class); + updateProcessingPackagesService.run(); + monitoringService.run(); + updatingFlagsService.run(); } } diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index 0f3ca72..61dc791 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -15,8 +15,8 @@ import ru.nbch.credit_tracker.entities.external_request.signals.Signals; import ru.nbch.credit_tracker.entities.external_response.monitoring_report_error.MonitoringReportErrors; import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest; import ru.nbch.credit_tracker.enums.Status; -import ru.nbch.credit_tracker.mapper.signal.PhoneFidMapMapper; import ru.nbch.credit_tracker.service.indic.FlagsService; +import ru.nbch.credit_tracker.service.indic.FlagsService2; import ru.nbch.credit_tracker.service.indic.PersonService; import ru.nbch.credit_tracker.service.indic.PhoneService; import ru.nbch.credit_tracker.service.signal.HitsService; @@ -44,9 +44,8 @@ public class SignalService { private final HitsService hitsService; private final UserPackageService userPackageService; private final PhoneFidMapService phoneFidMapService; - private final PhoneFidMapMapper phoneFidMapMapper; private final PhoneService phoneService; - private final FlagsService flagsService; + private final FlagsService2 flagsService; private final PersonService personService; private final StaxXmlProcessor staxXmlProcessor; @@ -56,16 +55,14 @@ public class SignalService { HitsService hitsService, UserPackageService userPackageService, PhoneFidMapService phoneFidMapService, - PhoneFidMapMapper mapper, PhoneService phoneService, - FlagsService flagsService, + FlagsService2 flagsService, PersonService personService, StaxXmlProcessor staxXmlProcessor) { this.config = config; this.hitsService = hitsService; this.userPackageService = userPackageService; this.phoneFidMapService = phoneFidMapService; - this.phoneFidMapMapper = mapper; this.phoneService = phoneService; this.flagsService = flagsService; this.personService = personService; diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java index 720f8e3..629b181 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java @@ -91,7 +91,7 @@ public class FlagsService2 { * @param flag90Days - рещультирующая мапа флагов просрочек 90_days */ private void findFlagsRuPastDueArrear(List fids, Map flag5000Map, Map flag90Days) { - log.debug("Start calculating flag 5000 and 90 days by RU_PASTDUEARREAR 2"); + log.debug("Start calculating flag 5000 and 90 days by RU_PASTDUEARREAR"); MapSqlParameterSource params = new MapSqlParameterSource(); params.addValue("fids", fids); @@ -167,7 +167,7 @@ public class FlagsService2 { } return null; }); - log.debug("Complete calculating flag 5000 and 90 days by RU_PASTDUEARREAR 2"); + log.debug("Complete calculating flag 5000 and 90 days by RU_PASTDUEARREAR"); } /** @@ -181,7 +181,7 @@ public class FlagsService2 { * @param flag90Days - рещультирующая мапа флагов просрочек 90_days */ private void findFlagsRuPaymt(List fids, Map flag90Days) { - log.debug("Start calculating flag 90 days by RU_PAPAYMT 2"); + log.debug("Start calculating flag 90 days by RU_PAPAYMT"); LocalDate yearAgo = LocalDate.now().minusYears(1); MapSqlParameterSource params = new MapSqlParameterSource(); params.addValue("fids", fids); @@ -233,7 +233,7 @@ public class FlagsService2 { } return null; }); - log.debug("Complete calculating flag 90 days by RU_PAPAYMT 2"); + log.debug("Complete calculating flag 90 days by RU_PAPAYMT"); } From 7a4695281ef934ce9199b1b90ef9262a979e8826 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Mon, 29 Sep 2025 13:12:00 +0300 Subject: [PATCH 19/20] Removed unused methods; Uncommented schedulers --- .../mapper/signal/PhoneFidMapMapper.java | 1 - .../service/MonitoringService.java | 37 ++++++++++--------- .../UpdateProcessingPackagesService.java | 17 +++++---- .../service/UpdatingFlagsService.java | 17 ++++++--- .../service/indic/PhoneService.java | 25 +------------ .../mapper/signal/PhoneFidMapMapper.xml | 11 ------ 6 files changed, 41 insertions(+), 67 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java b/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java index 80f216f..0e42995 100644 --- a/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java +++ b/src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java @@ -15,7 +15,6 @@ public interface PhoneFidMapMapper { List findFullPhoneFidMap(@Param("packageId") Integer packageId, @Param("offset") int offset, @Param("limit") int limit); void updatePhoneFidMaps(@Param("subjects") List subjects); - void updatePhoneFid(@Param("subject") PhoneFidMap subject); void updatePhoneFidMapsErrors(@Param("subjects") List subjects); diff --git a/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java b/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java index 1a8e25f..1f30550 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java @@ -1,22 +1,8 @@ package ru.nbch.credit_tracker.service; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; import ru.nbch.credit_tracker.entities.dto.Hit; @@ -36,6 +22,21 @@ import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlWr import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor; import ru.nbch.credit_tracker.utils.CTUtil; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + /** * Сервис периодического опроса официального сервиса сигналов на наличие новых отчетов по пакетам находящимся в статусе ON_MONITORING */ @@ -59,9 +60,9 @@ public class MonitoringService { } public void run() { -// ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-")); -// long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15; -// scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES); + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-")); + long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15; + scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES); } private void monitor() { diff --git a/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java b/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java index 5630238..2888c82 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java @@ -2,9 +2,12 @@ package ru.nbch.credit_tracker.service; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import ru.nbch.credit_tracker.entities.dto.UserPackage; import ru.nbch.credit_tracker.enums.Status; import ru.nbch.credit_tracker.service.signal.SignalRestService; +import java.util.List; + /** * Сервис для завершения обработки пакетов находящихся в статусе PROCESSING * Запускается при старте приложения и отрабатывает 1 раз @@ -26,13 +29,13 @@ public class UpdateProcessingPackagesService { } public void run() { -// log.info("Start updating processing packages"); -// List packages = signalService.findPackagesOnProcessing(); -// for (UserPackage userPackage : packages) { -// log.info("Updating flags for Package {}", userPackage.getPackageName()); -// process(userPackage.getId(), userPackage.getPackageName()); -// } -// log.info("Finish updating processing packages"); + log.info("Start updating processing packages"); + List packages = signalService.findPackagesOnProcessing(); + for (UserPackage userPackage : packages) { + log.info("Updating flags for Package {}", userPackage.getPackageName()); + process(userPackage.getId(), userPackage.getPackageName()); + } + log.info("Finish updating processing packages"); } private void process(Integer packageId, String packageName) { diff --git a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java index 6b0a373..19549fb 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java @@ -1,10 +1,7 @@ package ru.nbch.credit_tracker.service; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.function.Predicate; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; @@ -12,6 +9,14 @@ import ru.nbch.credit_tracker.entities.dto.UserPackage; import ru.nbch.credit_tracker.enums.Status; import ru.nbch.credit_tracker.service.signal.SignalRestService; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + /** * Сервис периодического обновления фидов и флагов просрочек по пакетам находящимся в статусе ON_MONITORING * По обновленным пакетам формируется новый запрос постановки на мониторинг в официальный сервис сигналов @@ -38,8 +43,8 @@ public class UpdatingFlagsService { } public void run() { -// ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("updating-flags-")); -// scheduler.scheduleAtFixedRate(this::update, 0, updateFlagsInterval, TimeUnit.DAYS); + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("updating-flags-")); + scheduler.scheduleAtFixedRate(this::update, 0, updateFlagsInterval, TimeUnit.DAYS); } private void update() { diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java index fa3b27c..7cd8a87 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java @@ -11,7 +11,6 @@ import ru.nbch.credit_tracker.mapper.indic.PhoneMapper; import java.time.LocalDate; import java.util.*; -import java.util.stream.Collectors; @Slf4j @Service @@ -25,13 +24,8 @@ public class PhoneService { /** * Ищет фиды по номерам телефона - * @return Map: key - номер телефона; value - fid + * @return Map: key - fid; value - {@link PhoneFidMap} */ - public Map findFidsPhoneData(List subjects) { - List phones = subjects.stream().map(subject -> "9" + subject.getPhone()).toList(); - return convertFidsDataToMap(mapper.findFids(phones)); - } - public Map defineFidByPhone(List subjects) { Map phoneFidMap = new HashMap<>(); List phones = subjects.stream(). @@ -65,23 +59,6 @@ public class PhoneService { return phoneFidMap; } - private Map convertFidsDataToMap(List fidsData) { - if (fidsData == null || fidsData.isEmpty()) { - return new HashMap<>(); - } - return fidsData.stream() - .collect(Collectors.groupingBy( - FidsData::getPhone, - Collectors.collectingAndThen( - Collectors.maxBy( - Comparator.comparing(FidsData::getFileSinceDate) - .thenComparing(FidsData::getFid) - ), - data -> data.map(FidsData::getFid).orElse(null) - ) - )); - } - @Getter @Setter @AllArgsConstructor diff --git a/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml b/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml index 3ff64c4..9d668d9 100644 --- a/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml +++ b/src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml @@ -42,17 +42,6 @@ - - UPDATE signals_phone_fid_map - SET - fid = #{subject.fid}, - flag_debt5000 = #{subject.flagDebt5000}, - flag_90_12 = #{subject.flag90days}, - calculated_at = #{subject.calculatedAt}, - error_text = #{subject.errorText} - WHERE id = #{subject.id} - - UPDATE signals_phone_fid_map From af824c7d07e9742b81a4c097009ef716c3f152c1 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Tue, 30 Sep 2025 10:59:26 +0300 Subject: [PATCH 20/20] Used jdbcTemplate for finding fids --- .../credit_tracker/service/SignalService.java | 6 +- .../service/indic/PhoneService2.java | 96 +++++++++++++++++++ .../service/PhoneService2Test.java | 63 ++++++++++++ 3 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java create mode 100644 src/test/java/ru/nbch/credit_tracker/service/PhoneService2Test.java diff --git a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java index 61dc791..53c418f 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/SignalService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/SignalService.java @@ -18,7 +18,7 @@ import ru.nbch.credit_tracker.enums.Status; import ru.nbch.credit_tracker.service.indic.FlagsService; import ru.nbch.credit_tracker.service.indic.FlagsService2; import ru.nbch.credit_tracker.service.indic.PersonService; -import ru.nbch.credit_tracker.service.indic.PhoneService; +import ru.nbch.credit_tracker.service.indic.PhoneService2; import ru.nbch.credit_tracker.service.signal.HitsService; import ru.nbch.credit_tracker.service.signal.PhoneFidMapService; import ru.nbch.credit_tracker.service.signal.UserPackageService; @@ -44,7 +44,7 @@ public class SignalService { private final HitsService hitsService; private final UserPackageService userPackageService; private final PhoneFidMapService phoneFidMapService; - private final PhoneService phoneService; + private final PhoneService2 phoneService; private final FlagsService2 flagsService; private final PersonService personService; private final StaxXmlProcessor staxXmlProcessor; @@ -55,7 +55,7 @@ public class SignalService { HitsService hitsService, UserPackageService userPackageService, PhoneFidMapService phoneFidMapService, - PhoneService phoneService, + PhoneService2 phoneService, FlagsService2 flagsService, PersonService personService, StaxXmlProcessor staxXmlProcessor) { diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java new file mode 100644 index 0000000..52f5a8c --- /dev/null +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java @@ -0,0 +1,96 @@ +package ru.nbch.credit_tracker.service.indic; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Service; +import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; + +import java.time.LocalDate; +import java.util.*; + +@Slf4j +@Service +public class PhoneService2 { + + private static final String SELCT_FIDS = """ + SELECT + pn.number, + pn.fid, + pn.file_since_dt + FROM PHONE_NORM pn + JOIN (SELECT DISTINCT fid FROM NAME) nf + ON nf.fid = pn.fid + WHERE number in (:phones) + ORDER BY + pn.number ASC, + pn.file_since_dt DESC, + pn.fid DESC + WITH UR + """; + + private final NamedParameterJdbcTemplate jdbcTemplate; + + public PhoneService2(@Qualifier("jdbcTemplateIndic") NamedParameterJdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + /** + * Ищет фиды по номерам телефона + * + * @return Map: key - fid; value - {@link PhoneFidMap} + */ + public Map defineFidByPhone(List subjects) { + Map phoneFidMap = new HashMap<>(); + + List phones = subjects.stream(). + map(subject -> "9" + subject.getPhone()) + .toList(); + + subjects.sort( + Comparator.comparing( + PhoneFidMap::getPhone, + Comparator.naturalOrder() + ) + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("phones", phones); + + List phonesByFid = jdbcTemplate.query(SELCT_FIDS, params, (rs, rowNum) -> + new FidsData(rs.getInt("fid"), rs.getString("number"), rs.getObject("file_since_dt", LocalDate.class)) + ); + + log.debug("Start merging fids"); + for (PhoneFidMap phoneFid : subjects) { + Iterator iterator = phonesByFid.iterator(); + while (iterator.hasNext()) { + FidsData fidsData = iterator.next(); + if (fidsData.getPhone().equals(9 + phoneFid.getPhone())) { + //fixme доступ по ссылке, страшно, придумать что-то еще + phoneFidMap.put(fidsData.getFid(), phoneFid); + iterator.remove(); + break; + } + } + } + log.debug("End merging fids"); + + return phoneFidMap; + } + + @Getter + @Setter + @AllArgsConstructor + @NoArgsConstructor + public static class FidsData { + private Integer fid; + private String phone; + private LocalDate fileSinceDate; + } +} diff --git a/src/test/java/ru/nbch/credit_tracker/service/PhoneService2Test.java b/src/test/java/ru/nbch/credit_tracker/service/PhoneService2Test.java new file mode 100644 index 0000000..a8f14c0 --- /dev/null +++ b/src/test/java/ru/nbch/credit_tracker/service/PhoneService2Test.java @@ -0,0 +1,63 @@ +package ru.nbch.credit_tracker.service; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; +import ru.nbch.credit_tracker.service.indic.PhoneService2; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class PhoneService2Test { + + @Mock + NamedParameterJdbcTemplate jdbcTemplate; + + @Test + public void defineFidByPhoneTest() { + PhoneService2 phoneService = new PhoneService2(jdbcTemplate); + + fidsDataWithEqualSinceDt.sort(byDbOrder); + when(jdbcTemplate.query(anyString(), any(MapSqlParameterSource.class), any(RowMapper.class))).thenReturn(fidsDataWithEqualSinceDt); + + List phoneFidMap = new ArrayList<>(); + phoneFidMap.add(PhoneFidMap.builder().phone("1111").build()); + phoneFidMap.add(PhoneFidMap.builder().phone("1112").build()); + phoneFidMap.add(PhoneFidMap.builder().phone("1113").build()); + phoneFidMap.add(PhoneFidMap.builder().phone("1114").build()); + + Map res = phoneService.defineFidByPhone(phoneFidMap); + Assertions.assertEquals(3, res.size()); + Assertions.assertEquals("1111", res.get(124).getPhone()); + Assertions.assertEquals("1112", res.get(125).getPhone()); + Assertions.assertEquals("1114", res.get(126).getPhone()); + } + + private final List fidsDataWithEqualSinceDt = new ArrayList<>( + List.of( + new PhoneService2.FidsData(123, "91111", LocalDate.of(2025, 9, 10)), + new PhoneService2.FidsData(124, "91111", LocalDate.of(2025, 9, 10)), + new PhoneService2.FidsData(125, "91112", LocalDate.of(2024, 10, 10)), + new PhoneService2.FidsData(126, "91114", LocalDate.of(2024, 10, 11)) + ) + ); + + Comparator byDbOrder = + Comparator.comparing(PhoneService2.FidsData::getPhone, Comparator.nullsFirst(Comparator.naturalOrder())) // pn.number ASC + .thenComparing(PhoneService2.FidsData::getFileSinceDate, Comparator.nullsLast(Comparator.reverseOrder())) // file_since_dt DESC + .thenComparing(PhoneService2.FidsData::getFid, Comparator.reverseOrder()); +}