treshenkov/credit_tracker
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix recalc flags

Browse source
This commit is contained in:
etreschenkov 2025-12-02 13:27:21 +03:00
parent a2ef593bc9
commit d78af08405
7 changed files with 28 additions and 292 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

243
src/main/java/ru/nbch/credit_tracker/service/SignalService.java
View file

@ -1,48 +1,25 @@
package ru.nbch.credit_tracker.service;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Predicate;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.builder.PersonFactoryBuilder;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.entities.app.Hit;
import ru.nbch.credit_tracker.entities.app.PhoneFidMap;
import ru.nbch.credit_tracker.entities.app.UserPackage;
import ru.nbch.credit_tracker.entities.external_request.signals.Auth;
import ru.nbch.credit_tracker.entities.external_request.signals.Person;
import ru.nbch.credit_tracker.entities.external_request.signals.Signals;
import ru.nbch.credit_tracker.entities.external_response.monitoring_report_error.MonitoringReportErrors;
import ru.nbch.credit_tracker.enums.Status;
import ru.nbch.credit_tracker.model.SubjectProfile;
import ru.nbch.credit_tracker.model.impl.SubjectPayload;
import ru.nbch.credit_tracker.service.indic.FlagsService;
import ru.nbch.credit_tracker.service.indic.PersonService;
import ru.nbch.credit_tracker.service.indic.PhoneService;
import ru.nbch.credit_tracker.service.indic.PhoneService2;
import ru.nbch.credit_tracker.service.signal.HitsService;
import ru.nbch.credit_tracker.service.signal.PhoneFidMapService;
import ru.nbch.credit_tracker.service.signal.UserPackageDao;
import ru.nbch.credit_tracker.service.task.impl.FidNameEnricher;
import ru.nbch.credit_tracker.service.task.impl.FlagsEnricher;
import ru.nbch.credit_tracker.service.xml.stax.SignalsStaxXmlWriter;
import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
import ru.nbch.credit_tracker.utils.CTUtil;
@Slf4j
@Service
@ -55,13 +32,7 @@ public class SignalService {
private final UserPackageDao userPackageService;
private final PhoneFidMapService phoneFidMapService;
private final PhoneService2 phoneService2;
private final PhoneService phoneService;
private final FlagsService flagsService;
private final PersonService personService;
private final StaxXmlProcessor staxXmlProcessor;
private final FidNameEnricher fidNameEnricher;
private final FlagsEnricher flagsEnricher;
private final int batchSize;
public SignalService(CreditTrackerSettings config,
@ -69,137 +40,14 @@ public class SignalService {
UserPackageDao userPackageService,
PhoneFidMapService phoneFidMapService,
PhoneService2 phoneService2,
PhoneService phoneService,
FlagsService flagsService,
PersonService personService,
StaxXmlProcessor staxXmlProcessor, FidNameEnricher fidNameEnricher, FlagsEnricher flagsEnricher) {
StaxXmlProcessor staxXmlProcessor) {
this.config = config;
this.hitsService = hitsService;
this.userPackageService = userPackageService;
this.phoneFidMapService = phoneFidMapService;
this.phoneService2 = phoneService2;
this.phoneService = phoneService;
this.flagsService = flagsService;
this.personService = personService;
this.staxXmlProcessor = staxXmlProcessor;
this.batchSize = config.getPipeline().subjectBatchSize();
this.fidNameEnricher = fidNameEnricher;
this.flagsEnricher = flagsEnricher;
}
public UpdateResult updateFlags2(UserPackage userPackage) {
log.debug("updating flags for package [{} {}]", userPackage.getPackageName(), userPackage.getId());
Long packageId = userPackage.getId();
int offset = 0;
List<PhoneFidMap> page = phoneService2.pageByPackageId(packageId, batchSize, offset);
int counter = 1;
while (!page.isEmpty()) {
log.debug("got page number {}, size {}", counter++, page.size());
List<SubjectProfile> subjectProfiles = page.stream().map(phoneFidMap -> SubjectProfile.builder()
.mapId(phoneFidMap.getId())
.identifier(phoneFidMap.getSubjectId())
.phone(phoneFidMap.getPhone())
.build()).toList();
SubjectPayload payload = new SubjectPayload(packageId, userPackage.getUserId(), userPackage.getPackageName(), subjectProfiles);
log.debug("page number {}: updating fids...", counter);
fidNameEnricher.handle(payload);
log.debug("page number {}: updating flags...", counter);
flagsEnricher.process(payload, true);
offset += batchSize;
log.debug("loading next page...");
page = phoneService2.pageByPackageId(packageId, batchSize, offset);
}
return new UpdateResult(true, false);
}
/**
* Обрабатывает все PhoneFidMap по packageId, выгружая их пачками и фильтруя по переданному фильтру
*/
public UpdateResult updateSubjects(Long packageId, Predicate<PhoneFidMap> filter) {
boolean updated = false;
boolean noFidsFound = true;
int offset = 0;
List<PhoneFidMap> batch;
Map<Integer, PhoneFidMap> phonesByFidToProcessing = new HashMap<>();
do {
batch = phoneFidMapService.findFullPhoneFidMap(packageId, offset, batchSize);
List<PhoneFidMap> filteredItems;
if (filter != null) {
filteredItems = batch.stream().filter(filter).toList();
} else {
filteredItems = batch;
}
if (!updated && !filteredItems.isEmpty()) {
updated = true;
}
if (!filteredItems.isEmpty()) {
Map<Integer, PhoneFidMap> fidsMapByBatch = phoneService.defineFidByPhone(filteredItems);
phoneService.defineFidByPhone(filteredItems);
log.debug("Actual fids size by phone's batch: {}", fidsMapByBatch.size());
if (fidsMapByBatch.size() + phonesByFidToProcessing.size() <= batchSize) {
phonesByFidToProcessing.putAll(fidsMapByBatch);
} else {
//todo execute find name and id -> forming xml
boolean fidsFound = updateFlags(phonesByFidToProcessing, packageId);
if (noFidsFound) {
noFidsFound = !fidsFound;
}
phonesByFidToProcessing = new HashMap<>(fidsMapByBatch);
}
}
offset += batchSize;
} while (!batch.isEmpty());
if (!phonesByFidToProcessing.isEmpty()) {
boolean fidsFound = updateFlags(phonesByFidToProcessing, packageId);
if (noFidsFound) {
noFidsFound = !fidsFound;
}
}
return new UpdateResult(updated, noFidsFound);
}
/**
* Обрабатывает пачку PhoneFidMap: обновляет фиды и флаги просрочек
* @return fidsFound true - если хотя бы по одному номеру был найден фид, false - в противном случае
*/
// TODO Добавить отдельный метод. Если не нашелся фид, надо обновить в базе fid = null и флаги = null calculated_at=now()
public boolean updateFlags(Map<Integer, PhoneFidMap> fidsMap, Long packageId) {
log.info("Start calculating flags for subjects batch. PackageId: {}", packageId);
Map<Integer, FlagsService.FlagsData> flagsDataMap = flagsService.findFlags(fidsMap.keySet().stream().toList());
List<PhoneFidMap> subjectToUpdate = new ArrayList<>();
for (Map.Entry<Integer, FlagsService.FlagsData> entry : flagsDataMap.entrySet()) {
Integer fid = entry.getKey();
FlagsService.FlagsData flagsData = entry.getValue();
PhoneFidMap phoneFidMap = fidsMap.get(fid);
phoneFidMap.setFid(fid);
phoneFidMap.setFlagDebt5000(flagsData.getFlagDebt5000());
phoneFidMap.setFlag90days(flagsData.getFlag90days());
phoneFidMap.setCalculatedAt(LocalDateTime.now());
subjectToUpdate.add(phoneFidMap);
}
log.debug("Start updating subject's batch");
if (!subjectToUpdate.isEmpty()) {
int updateBatchSize = 7000; // 9 параметров на одну запись для обновления. Максимальное количество параметров ~65000
for (int j = 0; j < subjectToUpdate.size(); j += updateBatchSize) {
int endIndex = Math.min(j + updateBatchSize, subjectToUpdate.size());
phoneFidMapService.updatePhoneFidMaps(subjectToUpdate.subList(j, endIndex));
log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjectToUpdate.size(), packageId);
}
}
log.debug("Complete updating subject's batch");
log.info("Finish calculating flags for subjects. PackageId: {}. Batch size: {}", packageId, fidsMap.size());
return !fidsMap.isEmpty();
}
@ -207,87 +55,6 @@ public class SignalService {
userPackageService.updatePackageStatus(packageId, status);
}
public void updatePackageOnMonitoringSuccess(Long packageId, LocalDateTime monitoringAt) {
userPackageService.updatePackageOnMonitoringSuccess(packageId, monitoringAt);
}
/**
* Формирует файл запроса в официальный сервис сигналов, выгружая паспортные данные пачками и сразу записывая их в файл
*
* @return Путь к файлу запроса к официальному сервису сигналов на постановку на мониторинг
*/
public String writeSignals(Long packageId) {
String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator;
String filePath = folderPath + UUID.randomUUID() + "_output.xml";
CTUtil.createFolder(folderPath);
Signals signals = buildSignals(packageId);
SignalsStaxXmlWriter signalsStaxXmlWriter = new SignalsStaxXmlWriter(config.getEncoding());
signalsStaxXmlWriter.writeWithStax(filePath,
signals,
writer -> buildPersons(packageId, writer));
return filePath;
}
public Signals buildSignals(Long packageId) {
UserPackage userPackage = userPackageService.findPackage(packageId);
Signals signals = new Signals();
signals.setOnline("2"); // Всегда=2 (только онлайн-сигналы)
signals.setProcessWithErrors("1"); // Всегда=1 (обработать пакет полностью, по обнаруженным ошибкам (при наличии) сформировать файл отказа, записи без ошибок установить на мониторинг)
signals.setId(userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName()); // <MEMBERCODE>_<PackageName>
Auth auth = new Auth();
auth.setUser(config.getSignalsuser());
signals.setAuth(auth);
return signals;
}
/**
* Обрабатывает все PhoneFidMap по packageId, выгружая их пачками и фильтруя по переданному фильтру.
* writer записывает пачку сразу в файл
*/
private void buildPersons(Long packageId, Consumer<List<Person>> writer) {
int offset = 0;
List<PhoneFidMap> batch;
do {
batch = phoneFidMapService.findFullPhoneFidMap(packageId, offset, batchSize);
List<Person> persons = findPersons(batch);
if (!persons.isEmpty()) {
writer.accept(persons); // записывает пачку Persons в файл
}
offset += batchSize;
} while (!batch.isEmpty());
}
/**
* Формирует список Person для постановки на мониторинг в официальный сервис сигналов на основе списка PhoneFidMap
*/
private List<Person> findPersons(List<PhoneFidMap> subjects) {
if (subjects.isEmpty()) {
return Collections.emptyList();
}
List<Person> persons = new ArrayList<>();
Map<Integer, PersonService.PersonData> personFidMap = personService.findPersonData(subjects.stream()
.map(PhoneFidMap::getFid)
.filter(Objects::nonNull).toList());
subjects.forEach(data -> {
if (data.getFid() != null) {
PersonService.PersonData personData = personFidMap.get(data.getFid());
Person person = PersonFactoryBuilder.builder()
.personData(personData)
.uid(data.getId())
.build();
if (person != null) {
persons.add(person);
}
}
});
return persons;
}
public void updateSubjectsErrors(Path filePath, Long packageId) {
log.info("Start updating subjects errors. PackageId: {}", packageId);
try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(filePath), 8 * 1024 * 1024)) {
@ -314,14 +81,6 @@ public class SignalService {
log.info("Subject's errors were updated. PackageId: {}", packageId);
}
public List<UserPackage> findPackagesOnMonitoring() {
return userPackageService.findAllPackagesByStatus(Status.ON_MONITORING);
}
public List<UserPackage> findPackagesOnProcessing() {
return userPackageService.findAllPackagesByStatus(Status.PROCESSING);
}
public void insertHits(List<Hit> hits) {
hitsService.insertHits(hits);
}

57
src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java
View file

@ -1,8 +1,7 @@
package ru.nbch.credit_tracker.service;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.concurrent.ExecutorService;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.MDC;
import org.springframework.scheduling.annotation.EnableScheduling;
@ -14,6 +13,10 @@ import ru.nbch.credit_tracker.enums.Status;
import ru.nbch.credit_tracker.mapper.signal.PhoneFidMapMapper;
import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
import ru.nbch.credit_tracker.service.signal.SignalRestService;
import ru.nbch.credit_tracker.service.task.Pipeline;
import ru.nbch.credit_tracker.service.task.PipelinePackageManager;
import ru.nbch.credit_tracker.service.task.SetupParam;
import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
/**
* Сервис периодического обновления фидов и флагов просрочек по пакетам находящимся в статусе ON_MONITORING
@ -30,66 +33,46 @@ public class UpdatingFlagsService {
private final Long daysToUpdateFlags;
private final PhoneFidMapMapper phoneFidMapMapper;
private final UserPackageMapper userPackageMapper;
private final PipelinePackageManager pipelinePackageManager;
private final ExecutorService pipelineExecutorService;
public UpdatingFlagsService(SignalService signalService,
SignalRestService signalRestService,
MonitoringErrorProcesingService monitoringErrorProcesingService,
CreditTrackerSettings config,
PhoneFidMapMapper phoneFidMapMapper, UserPackageMapper userPackageMapper) {
PhoneFidMapMapper phoneFidMapMapper, UserPackageMapper userPackageMapper,
PipelinePackageManager pipelinePackageManager,
ExecutorService pipelineExecutorService) {
this.signalService = signalService;
this.signalRestService = signalRestService;
this.monitoringErrorProcesingService = monitoringErrorProcesingService;
this.daysToUpdateFlags = config.getDaysToUpdateFlags() != null ? config.getDaysToUpdateFlags() : 1;
this.phoneFidMapMapper = phoneFidMapMapper;
this.userPackageMapper = userPackageMapper;
this.pipelinePackageManager = pipelinePackageManager;
this.pipelineExecutorService = pipelineExecutorService;
}
@Scheduled(cron = "${ct.update-flags-cron-expression}")
private void update() {
log.info("Start updating flags");
List<UserPackage> packages = signalService.findPackagesOnMonitoring();
List<UserPackage> packages = userPackageMapper.findAllPackagesByStatus(Status.ON_MONITORING.name());
for (UserPackage userPackage : packages) {
MDC.put("ru.nbch.credit_tracker.log.mdc.key", "[" + userPackage.getPackageName() + "]");
log.info("Check on updating flags for Package {}", userPackage.getPackageName());
boolean hasOldRow = phoneFidMapMapper.hasOldRow(userPackage.getId(), daysToUpdateFlags.intValue());
log.info("There are {} old rows", hasOldRow ? "exist" : "doesn't exist");
if (hasOldRow) {
updateFlags(userPackage);
try {
userPackageMapper.updatePackageStatus(userPackage.getId(), Status.PROCESSING_REPEATED.name());
Pipeline pipeline = pipelinePackageManager.buildAndRun(new SetupParam(userPackage.getId(), null, PipelineMode.DEFAULT));
pipeline.start(pipelineExecutorService);
} catch (Exception e) {
log.error("Failed to build pipeline", e);
}
}
MDC.remove("ru.nbch.credit_tracker.log.mdc.key");
}
log.info("Finish updating flags");
}
private void updateFlags(UserPackage userPackage) {
Long packageId = userPackage.getId();
String packageName = userPackage.getPackageName();
try {
//todo set reject_status = null
userPackageMapper.updatePackageStatus(packageId, Status.PROCESSING_REPEATED.name());
SignalService.UpdateResult updateResult = signalService.updateFlags2(userPackage);
if (updateResult.isUpdated()) {
if (signalRestService.setupMonitoring(packageId)) {
// LocalDateTime monitoringAt = userPackage.getMonitoringAt() != null ? userPackage.getMonitoringAt() : LocalDateTime.now();
log.info("Flags were updated successfully. PackageId: {} PackageName: {}. Package status changed to ON_MONITORING", packageId, packageName);
signalService.updatePackageOnMonitoringSuccess(packageId, LocalDateTime.now());
monitoringErrorProcesingService.findErrors(packageId);
} else {
log.info("Updated package registration was rejected by signal service. PackageId: {} PackageName: {}", packageId, packageName);
signalService.updatePackageStatus(packageId, Status.ERROR);
}
} else {
log.info("No flags were updated for package id {}. Package status changed to ON_MONITORING", packageId);
signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
}
} catch (Exception e) {
log.error("Ошибка при поиске фидов или перерасчете флагов просрочек. Package_id {}", packageId, e);
}
}
private boolean hasDaysPassed(LocalDateTime pastDateTime) {
LocalDateTime now = LocalDateTime.now();
long daysPassed = ChronoUnit.DAYS.between(pastDateTime, now);
return daysPassed >= daysToUpdateFlags;
}
}

2
src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java
View file

@ -31,6 +31,8 @@ public class PersonService {
return new HashMap<>();
}
Map<Integer, PersonData> personDataMap = new HashMap<>();
//делаем тоже самое что сделали signalService.updateFlags2(userPackage);
Map<Integer, NameData> nameDataMap = findNameData(fids);
Map<Integer, PassportData> passortDataMap = findPassportData(fids);

9
src/main/java/ru/nbch/credit_tracker/service/signal/SignalRestService.java
View file

@ -66,15 +66,6 @@ public class SignalRestService {
this.authToken = buildToken(config.getSignalsToken(), config.getSignalsuser());
}
/**
* Ставит пакет на мониторинг в официальный сервис сигналов
* @return true - если пакет успешно поставился на мониторинг. false - в противном случае
*/
public boolean setupMonitoring(Long packageId) throws Exception {
String path = signalService.writeSignals(packageId);
return setupMonitoring(Path.of(path));
}
public boolean setupMonitoring(Path signalFilePath) throws Exception {
Path gzip = ZipUtils.packGzip(signalFilePath);

4
src/main/java/ru/nbch/credit_tracker/service/task/impl/KickPipeLine.java
View file

@ -55,7 +55,7 @@ public abstract class KickPipeLine implements IPipeLineStarter<SubjectPayload> {
@Override
public void startFillingQueue(BlockingQueue<SubjectPayload> firstQueue) {
try {
log.info("Start registering subjects. PackageId: {};", packageId);
log.info("Start kicker: mode:{}, packageId: {};", mode(), packageId);
int limit = BATCH_SIZE;
UserPackage userPackage = userPackageMapper.findPackage(packageId);
List<PhoneFidMap> page = nextPage(limit);
@ -64,6 +64,8 @@ public abstract class KickPipeLine implements IPipeLineStarter<SubjectPayload> {
List<SubjectProfile> subjectProfiles = page.stream().map(phoneFidMap -> SubjectProfile.builder()
.mapId(phoneFidMap.getId())
.identifier(phoneFidMap.getSubjectId())
//наличие этого поля зависит от выборки, в определенных select намеренно опущено это поле
.fid(phoneFidMap.getFid())
.phone(phoneFidMap.getPhone())
.build()).toList();
try {

2
src/main/java/ru/nbch/credit_tracker/service/task/impl/KickPipeLineRestoreImpl.java
View file

@ -22,7 +22,7 @@ public class KickPipeLineRestoreImpl extends KickPipeLine {
@Override
protected PipelineMode mode() {
return PipelineMode.RESTORE;
return PipelineMode.REPEATED_RESTORE;
}
@Override

1
src/main/resources/mybatis/mapper/signal/PhoneFidMapMapper.xml
View file

@ -158,7 +158,6 @@
package_id,
subject_id,
phone,
fid,
error_text
FROM signals_phone_fid_map
WHERE package_id = #{packageId}