treshenkov/credit_tracker
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix fids finding

Browse source
This commit is contained in:
Mikhail Trofimov 2025-10-16 12:28:23 +03:00
parent 7fe066c42d
commit 1680abbec9
6 changed files with 108 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/main/java/ru/nbch/credit_tracker/model/FlagsPayload.java
View file

@ -3,22 +3,22 @@ package ru.nbch.credit_tracker.model;
import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
import ru.nbch.credit_tracker.service.task.TaskPayloadV2;
import java.util.Map;
import java.util.List;
public class FlagsPayload implements TaskPayloadV2 {
private final Integer packageId;
private final Map<Integer, PhoneFidMap> fidsMapByBatch;
private final List<PhoneFidMap> phoneFidMaps;
private final boolean poison;
public FlagsPayload(Integer packageId, Map<Integer, PhoneFidMap> fidsMapByBatch) {
public FlagsPayload(Integer packageId, List<PhoneFidMap> fidsMapByBatch) {
this.packageId = packageId;
this.fidsMapByBatch = fidsMapByBatch;
this.phoneFidMaps = fidsMapByBatch;
this.poison = false;
}
public FlagsPayload(boolean poison) {
this.packageId = null;
this.fidsMapByBatch = null;
this.phoneFidMaps = null;
this.poison = poison;
}
@ -26,8 +26,8 @@ public class FlagsPayload implements TaskPayloadV2 {
return packageId;
}
public Map<Integer, PhoneFidMap> getFidsMapByBatch() {
return fidsMapByBatch;
public List<PhoneFidMap> getPhoneFidMaps() {
return phoneFidMaps;
}
@Override

17
src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java
View file

@ -13,10 +13,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
@Slf4j
@Service
@ -42,16 +39,22 @@ public class FlagsService2 {
this.jdbcTemplate = jdbcTemplate;
}
public void fillFlagsData(Map<Integer, PhoneFidMap> fidsMapByBatch) {
List<Integer> fids = fidsMapByBatch.keySet().stream().toList();
// TODO optimize
public void fillFlagsData(List<PhoneFidMap> subjects) {
List<Integer> fids = subjects.stream().map(PhoneFidMap::getFid).filter(Objects::nonNull).toList();
Map<Integer, FlagsService.FlagsData> fidsflagsDataMap = findFlags(fids);
fidsflagsDataMap.forEach((fid, data) -> {
PhoneFidMap phoneFidMap = fidsMapByBatch.get(fid);
subjects.stream().filter(s-> s.getFid() != null && s.getFid().equals(fid)).forEach(phoneFidMap -> {
phoneFidMap.setFlagDebt5000(data.getFlagDebt5000());
phoneFidMap.setFlag90days(data.getFlag90days());
phoneFidMap.setCalculatedAt(LocalDateTime.now());
});
});
subjects.stream().filter(s-> s.getFid() == null).forEach(phoneFidMap -> {
phoneFidMap.setCalculatedAt(LocalDateTime.now());
});
}
public Map<Integer, FlagsService.FlagsData> findFlags(List<Integer> fids) {

7
src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java
View file

@ -111,11 +111,16 @@ public class PersonService {
.fileSinceDate(passport.getFileSinceDate())
.build();
subjectProfile.setIdProfile(idProfile);
subjectProfile.setFilled(true);
subjectProfile.setFilled(isRequiredPassportDataPresent(passport));
}
});
}
private boolean isRequiredPassportDataPresent(PassportData passportData) {
boolean isDocTypeNotNull = StringUtils.isNotBlank(passportData.idType) || StringUtils.isNotBlank(passportData.idType2);
return StringUtils.isNotBlank(passportData.idNum) && isDocTypeNotNull;
}
private PassportData findPassport(List<PassportData> passports) {
if (passports == null || passports.isEmpty()) {
return null;

51
src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java
View file

@ -62,6 +62,57 @@ public class PhoneService2 {
return phoneFidMap;
}
public void defineFidsAndNameByPhone2(List<PhoneFidMap> subjects) {
List<String> phones = subjects.stream().
map(subject -> "9" + subject.getPhone())
.toList();
subjects.sort(
Comparator.comparing(
PhoneFidMap::getPhone,
Comparator.naturalOrder()
)
);
List<PhoneRepository.FidsDataWithName> phonesByFid = phoneRepository.findFidsWithName(phones);
log.debug("Start merging fids and names");
Map<Integer, List<PhoneRepository.FidsDataWithName>> fidFidsDataMap = phonesByFid.stream()
.collect(Collectors.groupingBy(PhoneRepository.FidsDataWithName::getFid));
Map<String, Integer> phoneToFidsMap = new HashMap<>();
for (PhoneRepository.FidsDataWithName fidsData : phonesByFid) {
if (!phoneToFidsMap.containsKey(fidsData.getPhone())) {
phoneToFidsMap.put(fidsData.getPhone(), fidsData.getFid());
}
}
for (PhoneFidMap phoneFid : subjects) {
String phone = "9" + phoneFid.getPhone();
Integer fid = phoneToFidsMap.get(phone);
if (fid != null) {
phoneFid.setFid(fid);
List<PhoneRepository.FidsDataWithName> fidsDataWithNames = fidFidsDataMap.get(fid);
if (fidsDataWithNames != null) {
PhoneRepository.FidsDataWithName nameData = defineNameData(fidsDataWithNames);
if (nameData != null) {
phoneFid.setFirstName(nameData.getFirstName());
phoneFid.setMiddleName(nameData.getMiddleName());
phoneFid.setLastName(nameData.getLastName());
phoneFid.setBirthDate(nameData.getBirthDate());
phoneFid.setFileSinceDate(nameData.getFileSinceDateName());
}
fidFidsDataMap.remove(fid);
}
phoneToFidsMap.remove(phone);
}
}
log.debug("End merging fids and names");
}
/**
* Ищет фиды по номерам телефона. Обогащает {@link PhoneFidMap} для найденных фидов данными из Name и фидом.
* @return Map: key - FID, value - {@link PhoneFidMap}

17
src/main/java/ru/nbch/credit_tracker/service/task/FidNameEnricherV2.java
View file

@ -1,6 +1,7 @@
package ru.nbch.credit_tracker.service.task;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.entities.dto.NameProfile;
import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
@ -12,7 +13,6 @@ import ru.nbch.credit_tracker.model.SubjectsProfilePayloadV2;
import ru.nbch.credit_tracker.service.indic.PhoneService2;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
@Service
@ -35,12 +35,15 @@ public class FidNameEnricherV2 implements DispatcherV2<PhoneFidPayloadV2, Subjec
List<PhoneFidMap> phoneFidMap = taskPayload.getPhoneFidsBatch();
log.debug("FidNameEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), phoneFidMap.size());
mapper.registerWithId(phoneFidMap); // надо зарегистрировать phoneFidMap'ы т.к далее id участвует в формированиии запроса в оф. сервис uid = id
Map<Integer, PhoneFidMap> fidsMapByBatch = phoneService2.defineFidsAndNameByPhone(phoneFidMap);
log.debug("Found {} fids by batch", fidsMapByBatch.size());
flagsPayloadQueue.add(new FlagsPayload(taskPayload.getPackageId(), fidsMapByBatch)); // TODO find out how to use it in other way
phoneService2.defineFidsAndNameByPhone2(phoneFidMap);
log.debug("Found {} fids by batch", phoneFidMap.stream().filter(s -> s.getFid() != null).count());
flagsPayloadQueue.add(new FlagsPayload(taskPayload.getPackageId(), phoneFidMap)); // TODO find out how to use it in other way
//todo по phoneFidMap может быть найден не полный набор фидов, поэтому стоит это контролировать
List<SubjectProfile> subjectProfiles = fidsMapByBatch.values().stream().map(pf -> {
List<SubjectProfile> subjectProfiles = phoneFidMap.stream()
.filter(subject -> subject.getFid() != null)
.filter(this::requiredFieldPresent) // TODO log.warn("Person does not have required fields. Fid {}", fid);
.map(pf -> {
SubjectProfile subjectProfile = new SubjectProfile();
subjectProfile.setId(pf.getId());
subjectProfile.setFid(pf.getFid());
@ -56,4 +59,8 @@ public class FidNameEnricherV2 implements DispatcherV2<PhoneFidPayloadV2, Subjec
return new SubjectsProfilePayloadV2(taskPayload.getPackageId(), subjectProfiles);
}
private boolean requiredFieldPresent(PhoneFidMap phoneFidMap) {
return !StringUtils.isAllBlank(phoneFidMap.getFirstName(), phoneFidMap.getLastName()) && phoneFidMap.getBirthDate() != null;
}
}

19
src/main/java/ru/nbch/credit_tracker/service/task/FlagsEnricher.java
View file

@ -9,7 +9,6 @@ import ru.nbch.credit_tracker.model.FlagsPayload;
import ru.nbch.credit_tracker.service.indic.FlagsService2;
import java.util.List;
import java.util.Map;
@Service
@Slf4j
@ -26,19 +25,17 @@ public class FlagsEnricher implements DispatcherV2<FlagsPayload, DummyPayload>{
@Override
public DummyPayload handle(FlagsPayload taskPayload) {
Map<Integer, PhoneFidMap> fidsMapByBatch = taskPayload.getFidsMapByBatch();
log.debug("FlagsEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), fidsMapByBatch.size());
flagsService.fillFlagsData(fidsMapByBatch);
List<PhoneFidMap> subjects = fidsMapByBatch.values().stream().toList();
List<PhoneFidMap> phoneFidMaps = taskPayload.getPhoneFidMaps();
log.debug("FlagsEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), phoneFidMaps.size());
flagsService.fillFlagsData(phoneFidMaps);
log.debug("Start updating subject's batch");
if (!subjects.isEmpty()) {
if (!phoneFidMaps.isEmpty()) {
int updateBatchSize = 7000;
for (int j = 0; j < subjects.size(); j += updateBatchSize) {
int endIndex = Math.min(j + updateBatchSize, subjects.size());
mapper.updatePhoneFidMaps(subjects.subList(j, endIndex));
log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjects.size(), taskPayload.getPackageId());
for (int j = 0; j < phoneFidMaps.size(); j += updateBatchSize) {
int endIndex = Math.min(j + updateBatchSize, phoneFidMaps.size());
mapper.updatePhoneFidMaps(phoneFidMaps.subList(j, endIndex));
log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, phoneFidMaps.size(), taskPayload.getPackageId());
}
}
log.debug("Complete updating subject's batch");