diff --git a/src/main/java/ru/nbch/credit_tracker/model/FlagsPayload.java b/src/main/java/ru/nbch/credit_tracker/model/FlagsPayload.java index 9c4b167..a7695b5 100644 --- a/src/main/java/ru/nbch/credit_tracker/model/FlagsPayload.java +++ b/src/main/java/ru/nbch/credit_tracker/model/FlagsPayload.java @@ -3,22 +3,22 @@ package ru.nbch.credit_tracker.model; import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; import ru.nbch.credit_tracker.service.task.TaskPayloadV2; -import java.util.Map; +import java.util.List; public class FlagsPayload implements TaskPayloadV2 { private final Integer packageId; - private final Map fidsMapByBatch; + private final List phoneFidMaps; private final boolean poison; - public FlagsPayload(Integer packageId, Map fidsMapByBatch) { + public FlagsPayload(Integer packageId, List fidsMapByBatch) { this.packageId = packageId; - this.fidsMapByBatch = fidsMapByBatch; + this.phoneFidMaps = fidsMapByBatch; this.poison = false; } public FlagsPayload(boolean poison) { this.packageId = null; - this.fidsMapByBatch = null; + this.phoneFidMaps = null; this.poison = poison; } @@ -26,8 +26,8 @@ public class FlagsPayload implements TaskPayloadV2 { return packageId; } - public Map getFidsMapByBatch() { - return fidsMapByBatch; + public List getPhoneFidMaps() { + return phoneFidMaps; } @Override diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java index 045003d..d9ff5c2 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java @@ -13,10 +13,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Slf4j @Service @@ -42,14 +39,20 @@ public class FlagsService2 { this.jdbcTemplate = jdbcTemplate; } - public void fillFlagsData(Map fidsMapByBatch) { - List fids = fidsMapByBatch.keySet().stream().toList(); + // TODO optimize + public void fillFlagsData(List subjects) { + List fids = subjects.stream().map(PhoneFidMap::getFid).filter(Objects::nonNull).toList(); Map fidsflagsDataMap = findFlags(fids); fidsflagsDataMap.forEach((fid, data) -> { - PhoneFidMap phoneFidMap = fidsMapByBatch.get(fid); - phoneFidMap.setFlagDebt5000(data.getFlagDebt5000()); - phoneFidMap.setFlag90days(data.getFlag90days()); + subjects.stream().filter(s-> s.getFid() != null && s.getFid().equals(fid)).forEach(phoneFidMap -> { + phoneFidMap.setFlagDebt5000(data.getFlagDebt5000()); + phoneFidMap.setFlag90days(data.getFlag90days()); + phoneFidMap.setCalculatedAt(LocalDateTime.now()); + }); + }); + + subjects.stream().filter(s-> s.getFid() == null).forEach(phoneFidMap -> { phoneFidMap.setCalculatedAt(LocalDateTime.now()); }); } diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java index f387882..c5d35cd 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java @@ -111,11 +111,16 @@ public class PersonService { .fileSinceDate(passport.getFileSinceDate()) .build(); subjectProfile.setIdProfile(idProfile); - subjectProfile.setFilled(true); + subjectProfile.setFilled(isRequiredPassportDataPresent(passport)); } }); } + private boolean isRequiredPassportDataPresent(PassportData passportData) { + boolean isDocTypeNotNull = StringUtils.isNotBlank(passportData.idType) || StringUtils.isNotBlank(passportData.idType2); + return StringUtils.isNotBlank(passportData.idNum) && isDocTypeNotNull; + } + private PassportData findPassport(List passports) { if (passports == null || passports.isEmpty()) { return null; diff --git a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java index 0d5daba..d6d7d21 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java +++ b/src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java @@ -62,6 +62,57 @@ public class PhoneService2 { return phoneFidMap; } + public void defineFidsAndNameByPhone2(List subjects) { + List phones = subjects.stream(). + map(subject -> "9" + subject.getPhone()) + .toList(); + + subjects.sort( + Comparator.comparing( + PhoneFidMap::getPhone, + Comparator.naturalOrder() + ) + ); + + List phonesByFid = phoneRepository.findFidsWithName(phones); + + log.debug("Start merging fids and names"); + Map> fidFidsDataMap = phonesByFid.stream() + .collect(Collectors.groupingBy(PhoneRepository.FidsDataWithName::getFid)); + + Map phoneToFidsMap = new HashMap<>(); + for (PhoneRepository.FidsDataWithName fidsData : phonesByFid) { + if (!phoneToFidsMap.containsKey(fidsData.getPhone())) { + phoneToFidsMap.put(fidsData.getPhone(), fidsData.getFid()); + } + } + + for (PhoneFidMap phoneFid : subjects) { + String phone = "9" + phoneFid.getPhone(); + Integer fid = phoneToFidsMap.get(phone); + if (fid != null) { + phoneFid.setFid(fid); + + List fidsDataWithNames = fidFidsDataMap.get(fid); + if (fidsDataWithNames != null) { + PhoneRepository.FidsDataWithName nameData = defineNameData(fidsDataWithNames); + + if (nameData != null) { + phoneFid.setFirstName(nameData.getFirstName()); + phoneFid.setMiddleName(nameData.getMiddleName()); + phoneFid.setLastName(nameData.getLastName()); + phoneFid.setBirthDate(nameData.getBirthDate()); + phoneFid.setFileSinceDate(nameData.getFileSinceDateName()); + } + + fidFidsDataMap.remove(fid); + } + phoneToFidsMap.remove(phone); + } + } + log.debug("End merging fids and names"); + } + /** * Ищет фиды по номерам телефона. Обогащает {@link PhoneFidMap} для найденных фидов данными из Name и фидом. * @return Map: key - FID, value - {@link PhoneFidMap} diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/FidNameEnricherV2.java b/src/main/java/ru/nbch/credit_tracker/service/task/FidNameEnricherV2.java index 69dee34..ed9be88 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/FidNameEnricherV2.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/FidNameEnricherV2.java @@ -1,6 +1,7 @@ package ru.nbch.credit_tracker.service.task; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.entities.dto.NameProfile; import ru.nbch.credit_tracker.entities.dto.PhoneFidMap; @@ -12,7 +13,6 @@ import ru.nbch.credit_tracker.model.SubjectsProfilePayloadV2; import ru.nbch.credit_tracker.service.indic.PhoneService2; import java.util.List; -import java.util.Map; import java.util.concurrent.BlockingQueue; @Service @@ -35,25 +35,32 @@ public class FidNameEnricherV2 implements DispatcherV2 phoneFidMap = taskPayload.getPhoneFidsBatch(); log.debug("FidNameEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), phoneFidMap.size()); mapper.registerWithId(phoneFidMap); // надо зарегистрировать phoneFidMap'ы т.к далее id участвует в формированиии запроса в оф. сервис uid = id - Map fidsMapByBatch = phoneService2.defineFidsAndNameByPhone(phoneFidMap); - log.debug("Found {} fids by batch", fidsMapByBatch.size()); - flagsPayloadQueue.add(new FlagsPayload(taskPayload.getPackageId(), fidsMapByBatch)); // TODO find out how to use it in other way + phoneService2.defineFidsAndNameByPhone2(phoneFidMap); + log.debug("Found {} fids by batch", phoneFidMap.stream().filter(s -> s.getFid() != null).count()); + flagsPayloadQueue.add(new FlagsPayload(taskPayload.getPackageId(), phoneFidMap)); // TODO find out how to use it in other way //todo по phoneFidMap может быть найден не полный набор фидов, поэтому стоит это контролировать - List subjectProfiles = fidsMapByBatch.values().stream().map(pf -> { - SubjectProfile subjectProfile = new SubjectProfile(); - subjectProfile.setId(pf.getId()); - subjectProfile.setFid(pf.getFid()); - subjectProfile.setNameProfile(NameProfile.builder() - .firstName(pf.getFirstName()) - .middleName(pf.getMiddleName()) - .lastName(pf.getLastName()) - .birthDate(pf.getBirthDate()) - .fileSinceDate(pf.getFileSinceDate()) - .build()); - return subjectProfile; - }).toList(); + List subjectProfiles = phoneFidMap.stream() + .filter(subject -> subject.getFid() != null) + .filter(this::requiredFieldPresent) // TODO log.warn("Person does not have required fields. Fid {}", fid); + .map(pf -> { + SubjectProfile subjectProfile = new SubjectProfile(); + subjectProfile.setId(pf.getId()); + subjectProfile.setFid(pf.getFid()); + subjectProfile.setNameProfile(NameProfile.builder() + .firstName(pf.getFirstName()) + .middleName(pf.getMiddleName()) + .lastName(pf.getLastName()) + .birthDate(pf.getBirthDate()) + .fileSinceDate(pf.getFileSinceDate()) + .build()); + return subjectProfile; + }).toList(); return new SubjectsProfilePayloadV2(taskPayload.getPackageId(), subjectProfiles); } + + private boolean requiredFieldPresent(PhoneFidMap phoneFidMap) { + return !StringUtils.isAllBlank(phoneFidMap.getFirstName(), phoneFidMap.getLastName()) && phoneFidMap.getBirthDate() != null; + } } diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/FlagsEnricher.java b/src/main/java/ru/nbch/credit_tracker/service/task/FlagsEnricher.java index bcd2b26..b65f1f7 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/FlagsEnricher.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/FlagsEnricher.java @@ -9,7 +9,6 @@ import ru.nbch.credit_tracker.model.FlagsPayload; import ru.nbch.credit_tracker.service.indic.FlagsService2; import java.util.List; -import java.util.Map; @Service @Slf4j @@ -26,19 +25,17 @@ public class FlagsEnricher implements DispatcherV2{ @Override public DummyPayload handle(FlagsPayload taskPayload) { - Map fidsMapByBatch = taskPayload.getFidsMapByBatch(); - log.debug("FlagsEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), fidsMapByBatch.size()); - flagsService.fillFlagsData(fidsMapByBatch); - - List subjects = fidsMapByBatch.values().stream().toList(); + List phoneFidMaps = taskPayload.getPhoneFidMaps(); + log.debug("FlagsEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), phoneFidMaps.size()); + flagsService.fillFlagsData(phoneFidMaps); log.debug("Start updating subject's batch"); - if (!subjects.isEmpty()) { + if (!phoneFidMaps.isEmpty()) { int updateBatchSize = 7000; - for (int j = 0; j < subjects.size(); j += updateBatchSize) { - int endIndex = Math.min(j + updateBatchSize, subjects.size()); - mapper.updatePhoneFidMaps(subjects.subList(j, endIndex)); - log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjects.size(), taskPayload.getPackageId()); + for (int j = 0; j < phoneFidMaps.size(); j += updateBatchSize) { + int endIndex = Math.min(j + updateBatchSize, phoneFidMaps.size()); + mapper.updatePhoneFidMaps(phoneFidMaps.subList(j, endIndex)); + log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, phoneFidMaps.size(), taskPayload.getPackageId()); } } log.debug("Complete updating subject's batch");