Fix fids finding

src/main/java/ru/nbch/credit_tracker/model/FlagsPayload.java

@@ -3,22 +3,22 @@ package ru.nbch.credit_tracker.model;
 import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
 import ru.nbch.credit_tracker.service.task.TaskPayloadV2;
 
-import java.util.Map;
+import java.util.List;
 
 public class FlagsPayload implements TaskPayloadV2 {
     private final Integer packageId;
-    private final Map<Integer, PhoneFidMap> fidsMapByBatch;
+    private final List<PhoneFidMap> phoneFidMaps;
     private final boolean poison;
 
-    public FlagsPayload(Integer packageId, Map<Integer, PhoneFidMap> fidsMapByBatch) {
+    public FlagsPayload(Integer packageId, List<PhoneFidMap> fidsMapByBatch) {
         this.packageId = packageId;
-        this.fidsMapByBatch = fidsMapByBatch;
+        this.phoneFidMaps = fidsMapByBatch;
         this.poison = false;
     }
 
     public FlagsPayload(boolean poison) {
         this.packageId = null;
-        this.fidsMapByBatch = null;
+        this.phoneFidMaps = null;
         this.poison = poison;
     }
 
@@ -26,8 +26,8 @@ public class FlagsPayload implements TaskPayloadV2 {
         return packageId;
     }
 
-    public Map<Integer, PhoneFidMap> getFidsMapByBatch() {
+    public List<PhoneFidMap> getPhoneFidMaps() {
-        return fidsMapByBatch;
+        return phoneFidMaps;
     }
 
     @Override

src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java

@@ -13,10 +13,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
-import java.util.Comparator;
+import java.util.*;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 @Slf4j
 @Service
@@ -42,16 +39,22 @@ public class FlagsService2 {
         this.jdbcTemplate = jdbcTemplate;
     }
 
-    public void fillFlagsData(Map<Integer, PhoneFidMap> fidsMapByBatch) {
+    // TODO optimize
-        List<Integer> fids = fidsMapByBatch.keySet().stream().toList();
+    public void fillFlagsData(List<PhoneFidMap> subjects) {
+        List<Integer> fids = subjects.stream().map(PhoneFidMap::getFid).filter(Objects::nonNull).toList();
         Map<Integer, FlagsService.FlagsData> fidsflagsDataMap = findFlags(fids);
 
         fidsflagsDataMap.forEach((fid, data) -> {
-            PhoneFidMap phoneFidMap = fidsMapByBatch.get(fid);
+            subjects.stream().filter(s-> s.getFid() != null && s.getFid().equals(fid)).forEach(phoneFidMap -> {
                 phoneFidMap.setFlagDebt5000(data.getFlagDebt5000());
                 phoneFidMap.setFlag90days(data.getFlag90days());
                 phoneFidMap.setCalculatedAt(LocalDateTime.now());
             });
+        });
+
+        subjects.stream().filter(s-> s.getFid() == null).forEach(phoneFidMap -> {
+            phoneFidMap.setCalculatedAt(LocalDateTime.now());
+        });
     }
 
     public Map<Integer, FlagsService.FlagsData> findFlags(List<Integer> fids) {

src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java

@@ -111,11 +111,16 @@ public class PersonService {
                     .fileSinceDate(passport.getFileSinceDate())
                     .build();
                 subjectProfile.setIdProfile(idProfile);
-                subjectProfile.setFilled(true);
+                subjectProfile.setFilled(isRequiredPassportDataPresent(passport));
             }
         });
     }
 
+    private boolean isRequiredPassportDataPresent(PassportData passportData) {
+        boolean isDocTypeNotNull = StringUtils.isNotBlank(passportData.idType) || StringUtils.isNotBlank(passportData.idType2);
+        return StringUtils.isNotBlank(passportData.idNum) && isDocTypeNotNull;
+    }
+
     private PassportData findPassport(List<PassportData> passports) {
         if (passports == null || passports.isEmpty()) {
             return null;

src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java

@@ -62,6 +62,57 @@ public class PhoneService2 {
         return phoneFidMap;
     }
 
+    public void  defineFidsAndNameByPhone2(List<PhoneFidMap> subjects) {
+        List<String> phones = subjects.stream().
+                map(subject -> "9" + subject.getPhone())
+                .toList();
+
+        subjects.sort(
+                Comparator.comparing(
+                        PhoneFidMap::getPhone,
+                        Comparator.naturalOrder()
+                )
+        );
+
+        List<PhoneRepository.FidsDataWithName> phonesByFid = phoneRepository.findFidsWithName(phones);
+
+        log.debug("Start merging fids and names");
+        Map<Integer, List<PhoneRepository.FidsDataWithName>> fidFidsDataMap = phonesByFid.stream()
+                .collect(Collectors.groupingBy(PhoneRepository.FidsDataWithName::getFid));
+
+        Map<String, Integer> phoneToFidsMap = new HashMap<>();
+        for (PhoneRepository.FidsDataWithName fidsData : phonesByFid) {
+            if (!phoneToFidsMap.containsKey(fidsData.getPhone())) {
+                phoneToFidsMap.put(fidsData.getPhone(), fidsData.getFid());
+            }
+        }
+
+        for (PhoneFidMap phoneFid : subjects) {
+            String phone = "9" + phoneFid.getPhone();
+            Integer fid = phoneToFidsMap.get(phone);
+            if (fid != null) {
+                phoneFid.setFid(fid);
+
+                List<PhoneRepository.FidsDataWithName> fidsDataWithNames = fidFidsDataMap.get(fid);
+                if (fidsDataWithNames != null) {
+                    PhoneRepository.FidsDataWithName nameData = defineNameData(fidsDataWithNames);
+
+                    if (nameData != null) {
+                        phoneFid.setFirstName(nameData.getFirstName());
+                        phoneFid.setMiddleName(nameData.getMiddleName());
+                        phoneFid.setLastName(nameData.getLastName());
+                        phoneFid.setBirthDate(nameData.getBirthDate());
+                        phoneFid.setFileSinceDate(nameData.getFileSinceDateName());
+                    }
+
+                    fidFidsDataMap.remove(fid);
+                }
+                phoneToFidsMap.remove(phone);
+            }
+        }
+        log.debug("End merging fids and names");
+    }
+
     /**
      * Ищет фиды по номерам телефона. Обогащает {@link PhoneFidMap} для найденных фидов данными из Name и фидом.
      * @return Map: key - FID, value - {@link PhoneFidMap}

src/main/java/ru/nbch/credit_tracker/service/task/FidNameEnricherV2.java

@@ -1,6 +1,7 @@
 package ru.nbch.credit_tracker.service.task;
 
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 import ru.nbch.credit_tracker.entities.dto.NameProfile;
 import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
@@ -12,7 +13,6 @@ import ru.nbch.credit_tracker.model.SubjectsProfilePayloadV2;
 import ru.nbch.credit_tracker.service.indic.PhoneService2;
 
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 
 @Service
@@ -35,12 +35,15 @@ public class FidNameEnricherV2 implements DispatcherV2<PhoneFidPayloadV2, Subjec
         List<PhoneFidMap> phoneFidMap = taskPayload.getPhoneFidsBatch();
         log.debug("FidNameEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), phoneFidMap.size());
         mapper.registerWithId(phoneFidMap); // надо зарегистрировать phoneFidMap'ы т.к далее id участвует в формированиии запроса в оф. сервис uid = id
-        Map<Integer, PhoneFidMap> fidsMapByBatch = phoneService2.defineFidsAndNameByPhone(phoneFidMap);
+        phoneService2.defineFidsAndNameByPhone2(phoneFidMap);
-        log.debug("Found {} fids by batch", fidsMapByBatch.size());
+        log.debug("Found {} fids by batch", phoneFidMap.stream().filter(s -> s.getFid() != null).count());
-        flagsPayloadQueue.add(new FlagsPayload(taskPayload.getPackageId(), fidsMapByBatch)); // TODO find out how to use it in other way
+        flagsPayloadQueue.add(new FlagsPayload(taskPayload.getPackageId(), phoneFidMap)); // TODO find out how to use it in other way
         //todo по phoneFidMap может быть найден не полный набор фидов, поэтому стоит это контролировать
 
-        List<SubjectProfile> subjectProfiles = fidsMapByBatch.values().stream().map(pf -> {
+        List<SubjectProfile> subjectProfiles = phoneFidMap.stream()
+                .filter(subject -> subject.getFid() != null)
+                .filter(this::requiredFieldPresent) // TODO log.warn("Person does not have required fields. Fid {}", fid);
+                .map(pf -> {
                     SubjectProfile subjectProfile = new SubjectProfile();
                     subjectProfile.setId(pf.getId());
                     subjectProfile.setFid(pf.getFid());
@@ -56,4 +59,8 @@ public class FidNameEnricherV2 implements DispatcherV2<PhoneFidPayloadV2, Subjec
 
         return new SubjectsProfilePayloadV2(taskPayload.getPackageId(), subjectProfiles);
     }
+
+    private boolean requiredFieldPresent(PhoneFidMap phoneFidMap) {
+        return !StringUtils.isAllBlank(phoneFidMap.getFirstName(), phoneFidMap.getLastName()) && phoneFidMap.getBirthDate() != null;
+    }
 }

src/main/java/ru/nbch/credit_tracker/service/task/FlagsEnricher.java

@@ -9,7 +9,6 @@ import ru.nbch.credit_tracker.model.FlagsPayload;
 import ru.nbch.credit_tracker.service.indic.FlagsService2;
 
 import java.util.List;
-import java.util.Map;
 
 @Service
 @Slf4j
@@ -26,19 +25,17 @@ public class FlagsEnricher implements DispatcherV2<FlagsPayload, DummyPayload>{
 
     @Override
     public DummyPayload handle(FlagsPayload taskPayload) {
-        Map<Integer, PhoneFidMap> fidsMapByBatch = taskPayload.getFidsMapByBatch();
+        List<PhoneFidMap> phoneFidMaps = taskPayload.getPhoneFidMaps();
-        log.debug("FlagsEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), fidsMapByBatch.size());
+        log.debug("FlagsEnricher handle by packageId: {}, size: {}", taskPayload.getPackageId(), phoneFidMaps.size());
-        flagsService.fillFlagsData(fidsMapByBatch);
+        flagsService.fillFlagsData(phoneFidMaps);
-
-        List<PhoneFidMap> subjects = fidsMapByBatch.values().stream().toList();
 
         log.debug("Start updating subject's batch");
-        if (!subjects.isEmpty()) {
+        if (!phoneFidMaps.isEmpty()) {
             int updateBatchSize = 7000;
-            for (int j = 0; j < subjects.size(); j += updateBatchSize) {
+            for (int j = 0; j < phoneFidMaps.size(); j += updateBatchSize) {
-                int endIndex = Math.min(j + updateBatchSize, subjects.size());
+                int endIndex = Math.min(j + updateBatchSize, phoneFidMaps.size());
-                mapper.updatePhoneFidMaps(subjects.subList(j, endIndex));
+                mapper.updatePhoneFidMaps(phoneFidMaps.subList(j, endIndex));
-                log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjects.size(), taskPayload.getPackageId());
+                log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, phoneFidMaps.size(), taskPayload.getPackageId());
             }
         }
         log.debug("Complete updating subject's batch");