LP: aAdded updating fids for lp; Added abstraction for PipelineOrchestrators

This commit is contained in:
Mikhail Trofimov 2026-07-03 15:27:24 +03:00
parent 9a9ba6b803
commit 0c977947cd
24 changed files with 637 additions and 201 deletions

View file

@ -82,8 +82,10 @@ ct.monitoring-interval=${MONITORING_INTERVAL:5}
ct.update-flags-cron-expression=${UPDATE_FLAGS_CRON_EXPRESSION:0 25 18 * * *} ct.update-flags-cron-expression=${UPDATE_FLAGS_CRON_EXPRESSION:0 25 18 * * *}
ct.new-flags-cron-expression=${NEW_FLAGS_CRON_EXPRESSION:0 25 18 * * *} ct.new-flags-cron-expression=${NEW_FLAGS_CRON_EXPRESSION:0 25 18 * * *}
ct.days-to-update-flags=${DAYS_TO_UPDATE_FLAGS:30} ct.days-to-update-flags=${DAYS_TO_UPDATE_FLAGS:30}
ct.update-fid-cron-expression=${UPDATE_FIDS_CRON_EXPRESSION:0 10 12 * * *} ct.update-np-fid-cron-expression=${UPDATE_FIDS_CRON_EXPRESSION:0 10 12 * * *}
ct.days-to-update-fids=${DAYS_TO_UPDATE_FIDS:30} ct.update-lp-fid-cron-expression=${UPDATE_FIDS_CRON_EXPRESSION:0 10 12 * * *}
ct.days-to-update-np-fids=${DAYS_TO_UPDATE_FIDS:30}
ct.days-to-update-lp-fids=${DAYS_TO_UPDATE_FIDS:30}
#kafka #kafka
ct.kafka-consumer-settings.topic=${CONSUMER_TOPIC:my-topic} ct.kafka-consumer-settings.topic=${CONSUMER_TOPIC:my-topic}

View file

@ -39,7 +39,8 @@ public class CreditTrackerSettings {
private Long monitoringInterval; private Long monitoringInterval;
private String updateFlagsCronExpression; private String updateFlagsCronExpression;
private Integer daysToUpdateFlags; private Integer daysToUpdateFlags;
private Long daysToUpdateFids; private Long daysToUpdateNpFids;
private Long daysToUpdateLpFids;
private String encoding; private String encoding;
private Integer saverSubjectBatchSize; private Integer saverSubjectBatchSize;
} }

View file

@ -29,10 +29,7 @@ import ru.nbch.credit_tracker.model.MonitoringValidationError;
import ru.nbch.credit_tracker.model.ValidationResult; import ru.nbch.credit_tracker.model.ValidationResult;
import ru.nbch.credit_tracker.service.UserPackageService; import ru.nbch.credit_tracker.service.UserPackageService;
import ru.nbch.credit_tracker.service.scoring.AuthorizationService; import ru.nbch.credit_tracker.service.scoring.AuthorizationService;
import ru.nbch.credit_tracker.service.task.LpPipelinePackageFabric; import ru.nbch.credit_tracker.service.task.*;
import ru.nbch.credit_tracker.service.task.NpPipelinePackageFabric;
import ru.nbch.credit_tracker.service.task.PipelinePackageOrchestrator;
import ru.nbch.credit_tracker.service.task.SetupParam;
import ru.nbch.credit_tracker.service.task.impl.PipelineMode; import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
import ru.nbch.credit_tracker.service.task.impl.Saver; import ru.nbch.credit_tracker.service.task.impl.Saver;
import ru.nbch.credit_tracker.utils.CtFileUtils; import ru.nbch.credit_tracker.utils.CtFileUtils;
@ -67,7 +64,8 @@ public class SignalFacade {
private final NpPipelinePackageFabric npPipelinePackageFabric; private final NpPipelinePackageFabric npPipelinePackageFabric;
private final LpPipelinePackageFabric lpPipelinePackageFabric; private final LpPipelinePackageFabric lpPipelinePackageFabric;
private final Saver saver; private final Saver saver;
private final PipelinePackageOrchestrator pipelineOrchestrator; private final NpPipelinePackageOrchestrator pipelineOrchestrator;
private final LpPipelinePackageOrchestrator lpPipelinePackageOrchestrator;
private final Pattern DOWNLOAD_FILE_PATTERN = Pattern.compile("^(?<member>[A-Za-z0-9]{6})_(?<package>[A-Za-z0-9_]+)_(?<ts>\\d{4}\\d{2}\\d{2}_\\d{2}\\d{2}\\d{2})(?:\\.xml)?\\.zip$"); private final Pattern DOWNLOAD_FILE_PATTERN = Pattern.compile("^(?<member>[A-Za-z0-9]{6})_(?<package>[A-Za-z0-9_]+)_(?<ts>\\d{4}\\d{2}\\d{2}_\\d{2}\\d{2}\\d{2})(?:\\.xml)?\\.zip$");
public SignalFacade(MonitoringRequestValidator monitoringRequestValidator, public SignalFacade(MonitoringRequestValidator monitoringRequestValidator,
@ -76,7 +74,8 @@ public class SignalFacade {
AuthorizationService authorizationService, AuthorizationService authorizationService,
NpPipelinePackageFabric npPipelinePackageManager, NpPipelinePackageFabric npPipelinePackageManager,
LpPipelinePackageFabric lpPipelinePackageFabric, LpPipelinePackageFabric lpPipelinePackageFabric,
Saver saver, PipelinePackageOrchestrator pipelineOrchestrator) { Saver saver, NpPipelinePackageOrchestrator pipelineOrchestrator,
LpPipelinePackageOrchestrator lpPipelinePackageOrchestrator) {
this.monitoringRequestValidator = monitoringRequestValidator; this.monitoringRequestValidator = monitoringRequestValidator;
this.config = config; this.config = config;
this.userPackageService = userPackageService; this.userPackageService = userPackageService;
@ -85,6 +84,7 @@ public class SignalFacade {
this.lpPipelinePackageFabric = lpPipelinePackageFabric; this.lpPipelinePackageFabric = lpPipelinePackageFabric;
this.saver = saver; this.saver = saver;
this.pipelineOrchestrator = pipelineOrchestrator; this.pipelineOrchestrator = pipelineOrchestrator;
this.lpPipelinePackageOrchestrator = lpPipelinePackageOrchestrator;
} }
public void setupMonitoringRequest(MultipartFile file) { public void setupMonitoringRequest(MultipartFile file) {
@ -160,7 +160,7 @@ public class SignalFacade {
}, pipeline, packageId); }, pipeline, packageId);
} else { } else {
Pipeline pipeline = lpPipelinePackageFabric.build(new SetupParam(packageId, 2, PipelineMode.DEFAULT)); Pipeline pipeline = lpPipelinePackageFabric.build(new SetupParam(packageId, 2, PipelineMode.DEFAULT));
pipelineOrchestrator.startOrDelay(() -> { lpPipelinePackageOrchestrator.startOrDelay(() -> {
saver.readAndSaveLpPackage(packageId, parsedFilePath); saver.readAndSaveLpPackage(packageId, parsedFilePath);
CtFileUtils.deleteQuietly(parsedFilePath); CtFileUtils.deleteQuietly(parsedFilePath);
}, pipeline, packageId); }, pipeline, packageId);

View file

@ -32,4 +32,12 @@ public interface LegalsMapMapper {
@Param("lastId") Long lastId); @Param("lastId") Long lastId);
List<LegalsMap> findCachedByInnAndRegNUm(@Param("json") String json); List<LegalsMap> findCachedByInnAndRegNUm(@Param("json") String json);
void updateSearchedAt(@Param("json") String json);
void resetFidAndFlagsByInnAndRegNum(@Param("json") String json);
void updateFidCachedData(@Param("json") String json);
void deleteUnusedMaps();
} }

View file

@ -19,6 +19,8 @@ public interface PackagesRecordsMapper {
List<Long> uniquePackageIdByPhones(@Param("json") String json); List<Long> uniquePackageIdByPhones(@Param("json") String json);
List<Long> uniquePackageIdByInnAndRegNum(@Param("json") String json);
List<UserPackagePhoneData> selectUserPackagePhoneData(@Param("json") String json); List<UserPackagePhoneData> selectUserPackagePhoneData(@Param("json") String json);
List<UserPackageLegalData> selectUserPackageLegalData(@Param("json") String json); List<UserPackageLegalData> selectUserPackageLegalData(@Param("json") String json);

View file

@ -4,8 +4,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.entities.app.LegalsMap;
import ru.nbch.credit_tracker.entities.app.SignalLegalFidMap; import ru.nbch.credit_tracker.entities.app.SignalLegalFidMap;
import ru.nbch.credit_tracker.mapper.signal.LegalsFidMapMapper; import ru.nbch.credit_tracker.mapper.signal.LegalsFidMapMapper;
import ru.nbch.credit_tracker.mapper.signal.LegalsMapMapper;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -15,11 +17,14 @@ import java.util.List;
public class LegalsMapService { public class LegalsMapService {
private final LegalsFidMapMapper legalsFidMapMapper; private final LegalsFidMapMapper legalsFidMapMapper;
private final LegalsMapMapper legalsMapMapper;
private final ObjectMapper entityToJsonMapper; private final ObjectMapper entityToJsonMapper;
public LegalsMapService(LegalsFidMapMapper legalsFidMapMapper, public LegalsMapService(LegalsFidMapMapper legalsFidMapMapper,
LegalsMapMapper legalsMapMapper,
ObjectMapper entityToJsonMapper) { ObjectMapper entityToJsonMapper) {
this.legalsFidMapMapper = legalsFidMapMapper; this.legalsFidMapMapper = legalsFidMapMapper;
this.legalsMapMapper = legalsMapMapper;
this.entityToJsonMapper = entityToJsonMapper; this.entityToJsonMapper = entityToJsonMapper;
} }
@ -37,4 +42,47 @@ public class LegalsMapService {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public void updateSearchedAt(List<LegalsMap> subjects) {
try {
String json = entityToJsonMapper.writeValueAsString(subjects.stream()
.map(phoneFidMap -> new HashMap<String, Object>() {{
put("inn", phoneFidMap.getInn());
put("reg_num", phoneFidMap.getRegNum());
}}).toList());
legalsMapMapper.updateSearchedAt(json);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
public void resetFidAndFlagsByInnAndRegNum(List<LegalsMap> subjects) {
try {
String json = entityToJsonMapper.writeValueAsString(subjects.stream()
.map(phoneFidMap -> new HashMap<String, Object>() {{
put("inn", phoneFidMap.getInn());
put("reg_num", phoneFidMap.getRegNum());
}}).toList());
legalsMapMapper.resetFidAndFlagsByInnAndRegNum(json);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
public void updateFidCachedData(List<LegalsMap> subjects) {
try {
String json = entityToJsonMapper.writeValueAsString(subjects.stream()
.map(phoneFidMap -> new HashMap<String, Object>() {{
put("inn", phoneFidMap.getInn());
put("reg_num", phoneFidMap.getRegNum());
put("fid", phoneFidMap.getFid());
}}).toList());
legalsMapMapper.updateFidCachedData(json);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
} }

View file

@ -9,7 +9,7 @@ import ru.nbch.credit_tracker.enums.Status;
import ru.nbch.credit_tracker.mapper.signal.PackagesRecordsMapper; import ru.nbch.credit_tracker.mapper.signal.PackagesRecordsMapper;
import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper; import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
import ru.nbch.credit_tracker.service.task.NpPipelinePackageFabric; import ru.nbch.credit_tracker.service.task.NpPipelinePackageFabric;
import ru.nbch.credit_tracker.service.task.PipelinePackageOrchestrator; import ru.nbch.credit_tracker.service.task.NpPipelinePackageOrchestrator;
import ru.nbch.credit_tracker.service.task.SetupParam; import ru.nbch.credit_tracker.service.task.SetupParam;
import ru.nbch.credit_tracker.service.task.impl.PipelineMode; import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
import ru.nbch.credit_tracker.service.task.impl.Saver; import ru.nbch.credit_tracker.service.task.impl.Saver;
@ -32,14 +32,14 @@ public class StartupRestoreService {
private final UserPackageMapper userPackageMapper; private final UserPackageMapper userPackageMapper;
private final PackagesRecordsMapper phoneFidMapMapper; private final PackagesRecordsMapper phoneFidMapMapper;
private final NpPipelinePackageFabric npPipelinePackageManager; private final NpPipelinePackageFabric npPipelinePackageManager;
private final PipelinePackageOrchestrator pipelineOrchestrator; private final NpPipelinePackageOrchestrator pipelineOrchestrator;
private final CreditTrackerSettings creditTrackerSettings; private final CreditTrackerSettings creditTrackerSettings;
private final Saver saver; private final Saver saver;
public StartupRestoreService(UserPackageMapper userPackageMapper, public StartupRestoreService(UserPackageMapper userPackageMapper,
PackagesRecordsMapper phoneFidMapMapper, PackagesRecordsMapper phoneFidMapMapper,
NpPipelinePackageFabric npPipelinePackageManager, NpPipelinePackageFabric npPipelinePackageManager,
PipelinePackageOrchestrator pipelineOrchestrator, NpPipelinePackageOrchestrator pipelineOrchestrator,
CreditTrackerSettings creditTrackerSettings, CreditTrackerSettings creditTrackerSettings,
Saver saver) { Saver saver) {
this.userPackageMapper = userPackageMapper; this.userPackageMapper = userPackageMapper;
@ -53,6 +53,7 @@ public class StartupRestoreService {
public void run() { public void run() {
acceptedRestore(); acceptedRestore();
processingRestore(); processingRestore();
// TODO LP
} }
private void acceptedRestore() { private void acceptedRestore() {

View file

@ -6,12 +6,10 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.entities.app.UserPackage; import ru.nbch.credit_tracker.entities.app.UserPackage;
import ru.nbch.credit_tracker.mapper.signal.LegalsMapMapper;
import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper; import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
import ru.nbch.credit_tracker.service.signal.PhonesMapService; import ru.nbch.credit_tracker.service.signal.PhonesMapService;
import ru.nbch.credit_tracker.service.task.NpPipelinePackageFabric; import ru.nbch.credit_tracker.service.task.*;
import ru.nbch.credit_tracker.service.task.PipelinePackageOrchestrator;
import ru.nbch.credit_tracker.service.task.PipelineRecalcPDOrchestrator;
import ru.nbch.credit_tracker.service.task.SetupParam;
import ru.nbch.credit_tracker.service.task.impl.PipelineMode; import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
import ru.nbch.pipeline.engine.core.Pipeline; import ru.nbch.pipeline.engine.core.Pipeline;
@ -23,27 +21,39 @@ import java.util.concurrent.CompletableFuture;
@EnableScheduling @EnableScheduling
public class UpdatingFidService { public class UpdatingFidService {
private final NpPipelinePackageFabric npPipelinePackageFabric; private final NpPipelinePackageFabric npPipelinePackageFabric;
private final PipelineRecalcPDOrchestrator pipelineRecalcPDOrchestrator; private final LpPipelinePackageFabric lpPipelinePackageFabric;
private final PipelinePackageOrchestrator pipelinePackageOrchestrator; private final NpPipelineRecalcOrchestrator npPipelineRecalcOrchestrator;
private final LpPipelineRecalcOrchestrator lpPipelineRecalcOrchestrator;
private final NpPipelinePackageOrchestrator npPipelinePackageOrchestrator;
private final LpPipelinePackageOrchestrator lpPipelinePackageOrchestrator;
private final UserPackageMapper userPackageMapper; private final UserPackageMapper userPackageMapper;
private final PhonesMapService phonesMapService; private final PhonesMapService phonesMapService;
private final LegalsMapMapper legalsMapMapper;
public UpdatingFidService(CreditTrackerSettings config, public UpdatingFidService(CreditTrackerSettings config,
NpPipelinePackageFabric npPipelinePackageFabric, NpPipelinePackageFabric npPipelinePackageFabric,
PipelineRecalcPDOrchestrator pipelineRecalcPDOrchestrator, LpPipelinePackageFabric lpPipelinePackageFabric,
PipelinePackageOrchestrator pipelinePackageOrchestrator, NpPipelineRecalcOrchestrator npPipelineRecalcOrchestrator,
LpPipelineRecalcOrchestrator lpPipelineRecalcOrchestrator,
NpPipelinePackageOrchestrator npPipelinePackageOrchestrator,
LpPipelinePackageOrchestrator lpPipelinePackageOrchestrator,
UserPackageMapper userPackageMapper, UserPackageMapper userPackageMapper,
PhonesMapService phonesMapService) { PhonesMapService phonesMapService,
LegalsMapMapper legalsMapMapper) {
this.npPipelinePackageFabric = npPipelinePackageFabric; this.npPipelinePackageFabric = npPipelinePackageFabric;
this.pipelineRecalcPDOrchestrator = pipelineRecalcPDOrchestrator; this.lpPipelinePackageFabric = lpPipelinePackageFabric;
this.pipelinePackageOrchestrator = pipelinePackageOrchestrator; this.npPipelineRecalcOrchestrator = npPipelineRecalcOrchestrator;
this.lpPipelineRecalcOrchestrator = lpPipelineRecalcOrchestrator;
this.npPipelinePackageOrchestrator = npPipelinePackageOrchestrator;
this.lpPipelinePackageOrchestrator = lpPipelinePackageOrchestrator;
this.userPackageMapper = userPackageMapper; this.userPackageMapper = userPackageMapper;
this.phonesMapService = phonesMapService; this.phonesMapService = phonesMapService;
this.legalsMapMapper = legalsMapMapper;
} }
@Scheduled(cron = "${ct.update-fid-cron-expression}") @Scheduled(cron = "${ct.update-np-fid-cron-expression}")
public void updateNp() { public void updateNp() {
if (!pipelineRecalcPDOrchestrator.checkPermits()) { if (!npPipelineRecalcOrchestrator.checkPermits()) {
log.info("The previous task has not yet been completed."); log.info("The previous task has not yet been completed.");
return; return;
} }
@ -52,7 +62,7 @@ public class UpdatingFidService {
log.info("Start updating fids and PD"); log.info("Start updating fids and PD");
try { try {
Pipeline pipeline = npPipelinePackageFabric.buildRecalculationPD(new SetupParam(null, 1, PipelineMode.RECALC_FID)); Pipeline pipeline = npPipelinePackageFabric.buildRecalculationPD(new SetupParam(null, 1, PipelineMode.RECALC_FID));
CompletableFuture<?> future = pipelineRecalcPDOrchestrator.startOrReject(pipeline); CompletableFuture<?> future = npPipelineRecalcOrchestrator.startOrReject(pipeline);
future.whenComplete((o, throwable) -> { future.whenComplete((o, throwable) -> {
try { try {
if (throwable != null) { if (throwable != null) {
@ -66,7 +76,7 @@ public class UpdatingFidService {
SetupParam setupParam = new SetupParam(packageId, 1, PipelineMode.RESEND); SetupParam setupParam = new SetupParam(packageId, 1, PipelineMode.RESEND);
try { try {
Pipeline makeXmlByCache = npPipelinePackageFabric.build(setupParam); Pipeline makeXmlByCache = npPipelinePackageFabric.build(setupParam);
pipelinePackageOrchestrator.startOrDelay(makeXmlByCache, packageId); npPipelinePackageOrchestrator.startOrDelay(makeXmlByCache, packageId);
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to build pipeline", e); log.error("Failed to build pipeline", e);
} }
@ -74,7 +84,7 @@ public class UpdatingFidService {
} finally { } finally {
log.info("pipeline finished"); log.info("pipeline finished");
pipeline.close(); pipeline.close();
pipelineRecalcPDOrchestrator.release(); npPipelineRecalcOrchestrator.release();
} }
}); });
} catch (Exception e) { } catch (Exception e) {
@ -82,8 +92,44 @@ public class UpdatingFidService {
} }
} }
@Scheduled(cron = "${ct.update-fid-cron-expression}") // TODO another property @Scheduled(cron = "${ct.update-lp-fid-cron-expression}")
public void updateLp() { public void updateLp() {
// TODO if (!lpPipelineRecalcOrchestrator.checkPermits()) {
log.info("The previous task has not yet been completed.");
return;
}
log.info("Delete unused phones");
legalsMapMapper.deleteUnusedMaps();
log.info("Start updating fids");
try {
Pipeline pipeline = lpPipelinePackageFabric.buildRecalculation(new SetupParam(null, 2, PipelineMode.RECALC_FID));
CompletableFuture<?> future = lpPipelineRecalcOrchestrator.startOrReject(pipeline);
future.whenComplete((o, throwable) -> {
try {
if (throwable != null) {
log.error(throwable.getMessage(), throwable);
return;
}
List<UserPackage> packagesRepeated = userPackageMapper.findForRepeatedPackages();
packagesRepeated.forEach(userPackage -> {
Long packageId = userPackage.getId();
SetupParam setupParam = new SetupParam(packageId, 2, PipelineMode.RESEND);
try {
Pipeline setupPipeline = lpPipelinePackageFabric.build(setupParam);
lpPipelinePackageOrchestrator.startOrDelay(setupPipeline, packageId);
} catch (Exception e) {
log.error("Failed to build pipeline", e);
}
});
} finally {
log.info("pipeline finished");
pipeline.close();
lpPipelineRecalcOrchestrator.release();
}
});
} catch (Exception e) {
log.error("Failed to build pipeline", e);
}
} }
} }

View file

@ -113,7 +113,7 @@ public class PackageRecordsService {
} }
} }
public List<Long> selectUniquePackageIds(List<Long> phones) { public List<Long> selectNpUniquePackageIds(List<Long> phones) {
try { try {
String json = entityToJsonMapper.writeValueAsString(phones.stream() String json = entityToJsonMapper.writeValueAsString(phones.stream()
.filter(Objects::nonNull) .filter(Objects::nonNull)
@ -126,6 +126,20 @@ public class PackageRecordsService {
} }
} }
public List<Long> selectLpUniquePackageIds(List<LegalsMap> legalsMaps) {
try {
List<InnRegnumPair> pairs = legalsMaps.stream()
.filter(Objects::nonNull)
.map(legalMap -> new InnRegnumPair(legalMap.getInn(), legalMap.getRegNum()))
.toList();
String json = entityToJsonMapper.writeValueAsString(pairs);
return packagesRecordsMapper.uniquePackageIdByInnAndRegNum(json);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
public List<UserPackagePhoneData> selectUserPackagePhoneData(Set<Long> phones) { public List<UserPackagePhoneData> selectUserPackagePhoneData(Set<Long> phones) {
try { try {
String json = entityToJsonMapper.writeValueAsString(phones.stream() String json = entityToJsonMapper.writeValueAsString(phones.stream()

View file

@ -144,7 +144,7 @@ public class LpFidEnricher implements Handler<SubjectPayload, SubjectPayload> {
@Override @Override
public int preferredConcurrency() { public int preferredConcurrency() {
return creditTrackerSettings.getPipeline().nameEnricherStageConcurrency(); return creditTrackerSettings.getLpPipeline().nameEnricherStageConcurrency();
} }
@Override @Override

View file

@ -7,6 +7,7 @@ import org.springframework.stereotype.Component;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings; import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.config.task.StageQueueFactory; import ru.nbch.credit_tracker.config.task.StageQueueFactory;
import ru.nbch.credit_tracker.model.impl.*; import ru.nbch.credit_tracker.model.impl.*;
import ru.nbch.credit_tracker.service.task.impl.LpFidUpdater;
import ru.nbch.pipeline.engine.core.Pipeline; import ru.nbch.pipeline.engine.core.Pipeline;
import ru.nbch.pipeline.engine.core.PipelineBackoff; import ru.nbch.pipeline.engine.core.PipelineBackoff;
import ru.nbch.pipeline.engine.core.setup.IPipeLineStarter; import ru.nbch.pipeline.engine.core.setup.IPipeLineStarter;
@ -25,6 +26,7 @@ public class LpPipelinePackageFabric implements InitializingBean {
private final StageQueueFactory<SubjectPayload> lpFinishSetupQueueFactory; private final StageQueueFactory<SubjectPayload> lpFinishSetupQueueFactory;
private final StageQueueFactory<VoidPayload> noOpQueueFactory; private final StageQueueFactory<VoidPayload> noOpQueueFactory;
private final LpFidEnricher lpFidEnricher; private final LpFidEnricher lpFidEnricher;
private final LpFidUpdater lpFidUpdater;
private final Supplier<IPipeLineTerminator<SubjectPayload, SignalPayload>> pipelineTerminatorFactory; private final Supplier<IPipeLineTerminator<SubjectPayload, SignalPayload>> pipelineTerminatorFactory;
private final Supplier<IPipeLineTerminator<VoidPayload, EmptyTerminatePayload>> noOpTerminatorFactory; private final Supplier<IPipeLineTerminator<VoidPayload, EmptyTerminatePayload>> noOpTerminatorFactory;
private final Function<SetupParam, IPipeLineStarter<SubjectPayload>> pipelineStarterFactory; private final Function<SetupParam, IPipeLineStarter<SubjectPayload>> pipelineStarterFactory;
@ -37,6 +39,7 @@ public class LpPipelinePackageFabric implements InitializingBean {
@Qualifier("lpFinishSetupQueueFactory") StageQueueFactory<SubjectPayload> lpFinishSetupQueueFactory, @Qualifier("lpFinishSetupQueueFactory") StageQueueFactory<SubjectPayload> lpFinishSetupQueueFactory,
@Qualifier("lpNoOpQueueFactory") StageQueueFactory<VoidPayload> lpNoOpQueueFactory, @Qualifier("lpNoOpQueueFactory") StageQueueFactory<VoidPayload> lpNoOpQueueFactory,
LpFidEnricher lpFidEnricher, LpFidEnricher lpFidEnricher,
LpFidUpdater lpFidUpdater,
Supplier<IPipeLineTerminator<SubjectPayload, SignalPayload>> pipelineTerminatorFactory, Supplier<IPipeLineTerminator<SubjectPayload, SignalPayload>> pipelineTerminatorFactory,
Supplier<IPipeLineTerminator<VoidPayload, EmptyTerminatePayload>> noOpTerminatorFactory, Supplier<IPipeLineTerminator<VoidPayload, EmptyTerminatePayload>> noOpTerminatorFactory,
Function<SetupParam, IPipeLineStarter<SubjectPayload>> pipelineStarterFactory, Function<SetupParam, IPipeLineStarter<SubjectPayload>> pipelineStarterFactory,
@ -47,6 +50,7 @@ public class LpPipelinePackageFabric implements InitializingBean {
this.lpFinishSetupQueueFactory = lpFinishSetupQueueFactory; this.lpFinishSetupQueueFactory = lpFinishSetupQueueFactory;
this.noOpQueueFactory = lpNoOpQueueFactory; this.noOpQueueFactory = lpNoOpQueueFactory;
this.lpFidEnricher = lpFidEnricher; this.lpFidEnricher = lpFidEnricher;
this.lpFidUpdater = lpFidUpdater;
this.pipelineTerminatorFactory = pipelineTerminatorFactory; this.pipelineTerminatorFactory = pipelineTerminatorFactory;
this.noOpTerminatorFactory = noOpTerminatorFactory; this.noOpTerminatorFactory = noOpTerminatorFactory;
this.pipelineStarterFactory = pipelineStarterFactory; this.pipelineStarterFactory = pipelineStarterFactory;
@ -69,4 +73,15 @@ public class LpPipelinePackageFabric implements InitializingBean {
public void afterPropertiesSet() throws Exception { public void afterPropertiesSet() throws Exception {
log.info("pipeline settings: {}", creditTrackerSettings.getLpPipeline()); log.info("pipeline settings: {}", creditTrackerSettings.getLpPipeline());
} }
public Pipeline buildRecalculation(SetupParam setupParam) throws Exception {
return PipeLineBuilderV2.first(lpFidUpdaterQueueFactory.getObject(), noOpQueueFactory.getObject())
.handler(lpFidUpdater)
.andThen()
.fillInitialQueue(pipelineStarterFactory.apply(setupParam))
.pollFinishingQueue(noOpTerminatorFactory.get())
.backoff(pipelineBackoff)
.context(PackageVariable.class)
.build();
}
} }

View file

@ -0,0 +1,26 @@
package ru.nbch.credit_tracker.service.task;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.config.settings.PipelineSettings;
import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
import java.util.concurrent.ExecutorService;
@Service
public class LpPipelinePackageOrchestrator extends PackageOrchestrator {
private final CreditTrackerSettings creditTrackerSettings;
public LpPipelinePackageOrchestrator(CreditTrackerSettings creditTrackerSettings,
UserPackageMapper userPackageMapper, // TODO
ExecutorService pipelineExecutorService, // TODO
ExecutorService saverExecutorService) { // TODO
super(userPackageMapper, pipelineExecutorService, saverExecutorService);
this.creditTrackerSettings = creditTrackerSettings;
}
@Override
protected PipelineSettings getPipeline() {
return creditTrackerSettings.getPipeline();
}
}

View file

@ -0,0 +1,24 @@
package ru.nbch.credit_tracker.service.task;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.config.settings.PipelineSettings;
import java.util.concurrent.ExecutorService;
@Service
public class LpPipelineRecalcOrchestrator extends RecalcOrchestrator{
private final CreditTrackerSettings creditTrackerSettings;
public LpPipelineRecalcOrchestrator(CreditTrackerSettings creditTrackerSettings,
ExecutorService pipelineExecutorService) { // TODO another executor
super(pipelineExecutorService);
this.creditTrackerSettings = creditTrackerSettings;
}
@Override
protected PipelineSettings getPipelineSettings() {
return creditTrackerSettings.getLpPipeline();
}
}

View file

@ -0,0 +1,26 @@
package ru.nbch.credit_tracker.service.task;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.config.settings.PipelineSettings;
import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
import java.util.concurrent.ExecutorService;
@Service
public class NpPipelinePackageOrchestrator extends PackageOrchestrator {
private final CreditTrackerSettings creditTrackerSettings;
public NpPipelinePackageOrchestrator(CreditTrackerSettings creditTrackerSettings,
UserPackageMapper userPackageMapper,
ExecutorService pipelineExecutorService,
ExecutorService saverExecutorService) {
super(userPackageMapper, pipelineExecutorService, saverExecutorService);
this.creditTrackerSettings = creditTrackerSettings;
}
@Override
protected PipelineSettings getPipeline() {
return creditTrackerSettings.getPipeline();
}
}

View file

@ -0,0 +1,23 @@
package ru.nbch.credit_tracker.service.task;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.config.settings.PipelineSettings;
import java.util.concurrent.ExecutorService;
@Service
public class NpPipelineRecalcOrchestrator extends RecalcOrchestrator {
private final CreditTrackerSettings creditTrackerSettings;
public NpPipelineRecalcOrchestrator(CreditTrackerSettings creditTrackerSettings,
ExecutorService pipelineExecutorService) {
super(pipelineExecutorService);
this.creditTrackerSettings = creditTrackerSettings;
}
@Override
protected PipelineSettings getPipelineSettings() {
return creditTrackerSettings.getPipeline();
}
}

View file

@ -0,0 +1,148 @@
package ru.nbch.credit_tracker.service.task;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.nbch.credit_tracker.config.settings.PipelineSettings;
import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
import ru.nbch.credit_tracker.model.impl.PackageVariable;
import ru.nbch.credit_tracker.model.impl.SignalPayload;
import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
import ru.nbch.credit_tracker.service.task.impl.SetupMonitoringPipelineTerminator;
import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
import ru.nbch.pipeline.engine.core.Pipeline;
import java.time.LocalDateTime;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public abstract class PackageOrchestrator {
private final Logger log = LoggerFactory.getLogger(getClass());
private final UserPackageMapper userPackageMapper;
private final ExecutorService pipelineExecutorService;
private final ExecutorService saverExecutorService;
private final Object lock = new Object();
private final java.util.Deque<PackageOrchestrator.Launch> pending = new java.util.ArrayDeque<>();
private final Semaphore semaphore;
public PackageOrchestrator(UserPackageMapper userPackageMapper,
ExecutorService pipelineExecutorService,
ExecutorService saverExecutorService) {
this.userPackageMapper = userPackageMapper;
this.pipelineExecutorService = pipelineExecutorService;
this.saverExecutorService = saverExecutorService;
this.semaphore = new Semaphore(getPipeline().parallelRequests());
}
private record Launch(Runnable before, Pipeline pipeline, Long packageId) {
}
public void startOrDelay(Pipeline pipeline, Long packageId) {
startOrDelay(null, pipeline, packageId);
}
public void startOrDelay(Runnable before, Pipeline pipeline, Long packageId) {
synchronized (lock) {
if (!semaphore.tryAcquire()) {
log.debug("couldn't acquire permit from semaphore. delaying task for {}...", packageId);
pending.addLast(new PackageOrchestrator.Launch(before, pipeline, packageId));
return;
}
log.info("throttler permits after acquire: {}", semaphore.availablePermits());
}
innerStartOrDelay(before, pipeline, packageId);
}
private void innerStartOrDelay(Runnable before, Pipeline pipeline, Long packageId) {
try {
if (before != null) {
CompletableFuture
.runAsync(before, saverExecutorService)
.handle((v, t) -> {
if (t != null) {
log.error("`before` failed: {}", ExceptionUtils.getStackTrace(t));
pipeline.close();
checkForDelayedTasks(); // передать слот дальше или освободить
} else {
pipeline.start(pipelineExecutorService);
addCallback(pipeline, packageId);
}
return null;
});
} else {
pipeline.start(pipelineExecutorService);
addCallback(pipeline, packageId);
}
} catch (Exception e) {
log.error(ExceptionUtils.getStackTrace(e));
pipeline.close();
checkForDelayedTasks();
}
}
private void addCallback(Pipeline pipeline, Long packageId) {
pipeline.getTerminator()
.doneFuture()
.orTimeout(getPipeline().globalTimeout().toMillis(), TimeUnit.MILLISECONDS)
.whenComplete((v, e) -> {
try {
long finishTime = System.currentTimeMillis();
log.info("pipeline time passed: {} ms", finishTime - pipeline.getStartTime());
SignalPayload res = (SignalPayload) v;
var ctx = pipeline.obtainContext(PackageVariable.class);
AtomicInteger cacheHits = ctx.get(PackageVariable.cacheHits);
AtomicInteger cacheMisses = ctx.get(PackageVariable.cacheMisses);
log.info(
"Cache stats: hits={}, misses={}, total={}",
cacheHits.get(),
cacheMisses.get(),
cacheHits.get() + cacheMisses.get());
if (e instanceof SetupMonitoringPipelineTerminator.PipelineTerminatorException) {
log.info("when complete: terminator was interrupted.");
return;
} else if (e != null) {
log.error(ExceptionUtils.getStackTrace(e));
return;
}
if (!res.hasFids()) {
log.info("No one fid was found by packageId : {}.", packageId);
userPackageMapper.updateNoFidMonitoring(packageId);
return;
}
PipelineMode mode = ctx.get(PackageVariable.mode);
if (mode != null && mode.equals(PipelineMode.DEFAULT)) {
userPackageMapper.updatePackageOnMonitoringSuccess(packageId, LocalDateTime.now());
} else {
userPackageMapper.updatePackageOnMonitoringSuccess(packageId, null);
}
} catch (Exception ex) {
log.error("pipeline finished with error{}", ExceptionUtils.getStackTrace(ex));
} finally {
log.info("packageId={}: pipeline finished", packageId);
pipeline.close();
checkForDelayedTasks();
log.info("semaphore amount: {}", semaphore.availablePermits());
}
});
}
private void checkForDelayedTasks() {
log.debug("checking for delayed pipelines...");
PackageOrchestrator.Launch delayedLaunch;
synchronized (lock) {
delayedLaunch = pending.pollFirst();
if (delayedLaunch == null) {
log.debug("no delayed pipelines found; releasing semaphore...");
semaphore.release();
return;
}
}
log.info("delayed pipeline found: packageId={}. continue work...", delayedLaunch.packageId);
innerStartOrDelay(delayedLaunch.before, delayedLaunch.pipeline, delayedLaunch.packageId);
}
protected abstract PipelineSettings getPipeline();
}

View file

@ -1,151 +0,0 @@
package ru.nbch.credit_tracker.service.task;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
import ru.nbch.credit_tracker.model.impl.PackageVariable;
import ru.nbch.credit_tracker.model.impl.SignalPayload;
import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
import ru.nbch.credit_tracker.service.task.impl.SetupMonitoringPipelineTerminator;
import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
import ru.nbch.pipeline.engine.core.Pipeline;
import java.time.LocalDateTime;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Service
public class PipelinePackageOrchestrator {
private final Logger log = LoggerFactory.getLogger(getClass());
private final CreditTrackerSettings creditTrackerSettings;
private final UserPackageMapper userPackageMapper;
private final ExecutorService pipelineExecutorService;
private final ExecutorService saverExecutorService;
private final Object lock = new Object();
private final java.util.Deque<Launch> pending = new java.util.ArrayDeque<>();
private final Semaphore semaphore;
public PipelinePackageOrchestrator(CreditTrackerSettings creditTrackerSettings,
UserPackageMapper userPackageMapper,
ExecutorService pipelineExecutorService,
ExecutorService saverExecutorService) {
this.creditTrackerSettings = creditTrackerSettings;
this.userPackageMapper = userPackageMapper;
this.pipelineExecutorService = pipelineExecutorService;
this.saverExecutorService = saverExecutorService;
this.semaphore = new Semaphore(creditTrackerSettings.getPipeline().parallelRequests());
}
private record Launch(Runnable before, Pipeline pipeline, Long packageId) {
}
public void startOrDelay(Pipeline pipeline, Long packageId) {
startOrDelay(null, pipeline, packageId);
}
public void startOrDelay(Runnable before, Pipeline pipeline, Long packageId) {
synchronized (lock) {
if (!semaphore.tryAcquire()) {
log.debug("couldn't acquire permit from semaphore. delaying task for {}...", packageId);
pending.addLast(new Launch(before, pipeline, packageId));
return;
}
log.info("throttler permits after acquire: {}", semaphore.availablePermits());
}
innerStartOrDelay(before, pipeline, packageId);
}
private void innerStartOrDelay(Runnable before, Pipeline pipeline, Long packageId) {
try {
if (before != null) {
CompletableFuture
.runAsync(before, saverExecutorService)
.handle((v, t) -> {
if (t != null) {
log.error("`before` failed: {}", ExceptionUtils.getStackTrace(t));
pipeline.close();
checkForDelayedTasks(); // передать слот дальше или освободить
} else {
pipeline.start(pipelineExecutorService);
addCallback(pipeline, packageId);
}
return null;
});
} else {
pipeline.start(pipelineExecutorService);
addCallback(pipeline, packageId);
}
} catch (Exception e) {
log.error(ExceptionUtils.getStackTrace(e));
pipeline.close();
checkForDelayedTasks();
}
}
private void addCallback(Pipeline pipeline, Long packageId) {
pipeline.getTerminator()
.doneFuture()
.orTimeout(creditTrackerSettings.getPipeline().globalTimeout().toMillis(), TimeUnit.MILLISECONDS)
.whenComplete((v, e) -> {
try {
long finishTime = System.currentTimeMillis();
log.info("pipeline time passed: {} ms", finishTime - pipeline.getStartTime());
SignalPayload res = (SignalPayload) v;
var ctx = pipeline.obtainContext(PackageVariable.class);
AtomicInteger cacheHits = ctx.get(PackageVariable.cacheHits);
AtomicInteger cacheMisses = ctx.get(PackageVariable.cacheMisses);
log.info(
"Cache stats: hits={}, misses={}, total={}",
cacheHits.get(),
cacheMisses.get(),
cacheHits.get() + cacheMisses.get());
if (e instanceof SetupMonitoringPipelineTerminator.PipelineTerminatorException) {
log.info("when complete: terminator was interrupted.");
return;
} else if (e != null) {
log.error(ExceptionUtils.getStackTrace(e));
return;
}
if (!res.hasFids()) {
log.info("No one fid was found by packageId : {}.", packageId);
userPackageMapper.updateNoFidMonitoring(packageId);
return;
}
PipelineMode mode = ctx.get(PackageVariable.mode);
if (mode != null && mode.equals(PipelineMode.DEFAULT)) {
userPackageMapper.updatePackageOnMonitoringSuccess(packageId, LocalDateTime.now());
} else {
userPackageMapper.updatePackageOnMonitoringSuccess(packageId, null);
}
} catch (Exception ex) {
log.error("pipeline finished with error{}", ExceptionUtils.getStackTrace(ex));
} finally {
log.info("packageId={}: pipeline finished", packageId);
pipeline.close();
checkForDelayedTasks();
log.info("semaphore amount: {}", semaphore.availablePermits());
}
});
}
private void checkForDelayedTasks() {
log.debug("checking for delayed pipelines...");
Launch delayedLaunch;
synchronized (lock) {
delayedLaunch = pending.pollFirst();
if (delayedLaunch == null) {
log.debug("no delayed pipelines found; releasing semaphore...");
semaphore.release();
return;
}
}
log.info("delayed pipeline found: packageId={}. continue work...", delayedLaunch.packageId);
innerStartOrDelay(delayedLaunch.before, delayedLaunch.pipeline, delayedLaunch.packageId);
}
}

View file

@ -2,8 +2,7 @@ package ru.nbch.credit_tracker.service.task;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import ru.nbch.credit_tracker.config.settings.PipelineSettings;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.utils.error.ExceptionUtils; import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
import ru.nbch.pipeline.engine.core.Pipeline; import ru.nbch.pipeline.engine.core.Pipeline;
@ -12,17 +11,13 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@Service public abstract class RecalcOrchestrator {
public class PipelineRecalcPDOrchestrator {
private final Logger log = LoggerFactory.getLogger(getClass()); private final Logger log = LoggerFactory.getLogger(getClass());
private final CreditTrackerSettings creditTrackerSettings;
private final ExecutorService pipelineExecutorService; private final ExecutorService pipelineExecutorService;
private final Object lock = new Object(); private final Object lock = new Object();
private final Semaphore semaphore; private final Semaphore semaphore;
public PipelineRecalcPDOrchestrator(CreditTrackerSettings creditTrackerSettings, public RecalcOrchestrator(ExecutorService pipelineExecutorService) {
ExecutorService pipelineExecutorService) {
this.creditTrackerSettings = creditTrackerSettings;
this.pipelineExecutorService = pipelineExecutorService; this.pipelineExecutorService = pipelineExecutorService;
this.semaphore = new Semaphore(1); this.semaphore = new Semaphore(1);
} }
@ -43,6 +38,7 @@ public class PipelineRecalcPDOrchestrator {
return semaphore.availablePermits() > 0; return semaphore.availablePermits() > 0;
} }
} }
private CompletableFuture<?> innerStart(Pipeline pipeline) { private CompletableFuture<?> innerStart(Pipeline pipeline) {
try { try {
pipeline.start(pipelineExecutorService); pipeline.start(pipelineExecutorService);
@ -57,8 +53,8 @@ public class PipelineRecalcPDOrchestrator {
private CompletableFuture<?> addCallback(Pipeline pipeline) { private CompletableFuture<?> addCallback(Pipeline pipeline) {
return pipeline.getTerminator() return pipeline.getTerminator()
.doneFuture() .doneFuture()
.orTimeout(creditTrackerSettings.getPipeline().globalTimeout().toMillis(), TimeUnit.MILLISECONDS); .orTimeout(getPipelineSettings().globalTimeout().toMillis(), TimeUnit.MILLISECONDS);
} }
public void release() { public void release() {
@ -68,4 +64,6 @@ public class PipelineRecalcPDOrchestrator {
log.info("semaphore amount: {}", semaphore.availablePermits()); log.info("semaphore amount: {}", semaphore.availablePermits());
} }
} }
protected abstract PipelineSettings getPipelineSettings();
} }

View file

@ -0,0 +1,136 @@
package ru.nbch.credit_tracker.service.task.impl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.entities.app.LegalsMap;
import ru.nbch.credit_tracker.model.FidIdData;
import ru.nbch.credit_tracker.model.SubjectProfile;
import ru.nbch.credit_tracker.model.impl.PackageVariable;
import ru.nbch.credit_tracker.model.impl.SubjectPayload;
import ru.nbch.credit_tracker.model.impl.VoidPayload;
import ru.nbch.credit_tracker.service.UserPackageService;
import ru.nbch.credit_tracker.service.indic.LegalsMapService;
import ru.nbch.credit_tracker.service.indic.PersonService;
import ru.nbch.credit_tracker.service.signal.PackageRecordsService;
import ru.nbch.credit_tracker.utils.CTUtil;
import ru.nbch.pipeline.engine.core.Handler;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@Service
@Slf4j
public class LpFidUpdater implements Handler<SubjectPayload, VoidPayload> {
private final PersonService personService;
private final CreditTrackerSettings creditTrackerSettings;
private final PackageRecordsService packageRecordsService;
private final LegalsMapService legalsMapService;
private final UserPackageService userPackageService;
public LpFidUpdater(PersonService personService,
CreditTrackerSettings creditTrackerSettings,
PackageRecordsService packageRecordsService,
LegalsMapService legalsMapService,
UserPackageService userPackageService) {
this.personService = personService;
this.creditTrackerSettings = creditTrackerSettings;
this.packageRecordsService = packageRecordsService;
this.legalsMapService = legalsMapService;
this.userPackageService = userPackageService;
}
@Override
public VoidPayload handle(SubjectPayload taskPayload) {
List<SubjectProfile> subjects = taskPayload.getSubjects();
var ctx = obtainContext(PackageVariable.class, taskPayload);
log.debug("LpFidUpdaterStage handle, size of bucket for save: {}", subjects.size());
Map<String, List<FidIdData>> fidsByInnMap = personService.findFidsByInn(subjects);
Map<String, List<FidIdData>> fidsByRegNumMap = personService.findFidsByRegNum(subjects);
List<LegalsMap> changingFidLegalMaps = new ArrayList<>();
List<LegalsMap> notFoundFidLegalMaps = new ArrayList<>();
List<LegalsMap> notChangingFidLegalMaps = new ArrayList<>();
for (SubjectProfile subject : subjects) {
Long inn = subject.getInn();
Long regNum = subject.getRegNum();
List<FidIdData> fidByInn = fidsByInnMap.get(CTUtil.normalizeInnZeroesOnly(inn));
List<FidIdData> fidByRegNum = fidsByRegNumMap.get(CTUtil.normalizeRegNumZeroesOnly(regNum));
if (fidByInn == null || fidByInn.isEmpty()) {
// fid не найден
notFoundFidLegalMaps.add(fillLegalMap(subject, null));
continue;
}
List<Integer> fidsByInn = fidByInn.stream().map(FidIdData::getFid).toList();
if (fidByRegNum != null && !fidByRegNum.isEmpty()) {
List<Integer> fidsByRegNum = fidByRegNum.stream().map(FidIdData::getFid).toList();
// находим все пересечения по фидам
List<Integer> intersection = new ArrayList<>(fidsByInn);
intersection.retainAll(fidsByRegNum);
if (!intersection.isEmpty()) {
// берем максимальное пересечение
Integer maxFid = Collections.max(intersection);
if (maxFid.equals(subject.getFid())) {
notChangingFidLegalMaps.add(fillLegalMap(subject, maxFid));
} else {
changingFidLegalMaps.add(fillLegalMap(subject, maxFid));
}
continue;
}
}
notFoundFidLegalMaps.add(fillLegalMap(subject, null));
}
List<LegalsMap> changingFids = new ArrayList<>();
if (!notFoundFidLegalMaps.isEmpty()) {
legalsMapService.resetFidAndFlagsByInnAndRegNum(notFoundFidLegalMaps);
changingFids.addAll(notFoundFidLegalMaps);
}
if (!changingFidLegalMaps.isEmpty()) {
legalsMapService.updateFidCachedData(changingFidLegalMaps);
changingFids.addAll(changingFidLegalMaps);
}
if (!notChangingFidLegalMaps.isEmpty()) {
legalsMapService.updateSearchedAt(notChangingFidLegalMaps);
}
if (!changingFids.isEmpty()) {
List<Long> packageIds = packageRecordsService.selectLpUniquePackageIds(changingFids);
if (!packageIds.isEmpty()) {
userPackageService.setFidChanged(packageIds);
}
}
return new VoidPayload();
}
private LegalsMap fillLegalMap(SubjectProfile subject, Integer fid) {
return LegalsMap.builder()
.inn(subject.getInn())
.regNum(subject.getRegNum())
.fid(fid)
.build();
}
@Override
public int preferredConcurrency() {
return creditTrackerSettings.getLpPipeline().nameUpdaterStageConcurrency();
}
@Override
public VoidPayload cookPoison() {
return VoidPayload.POISON;
}
}

View file

@ -44,7 +44,7 @@ public class LpKickRecalcPipeLine implements IPipeLineStarter<SubjectPayload> {
this.packageRecordsService = packageRecordsService; this.packageRecordsService = packageRecordsService;
this.creditTrackerSettings = creditTrackerSettings; this.creditTrackerSettings = creditTrackerSettings;
this.BATCH_SIZE = creditTrackerSettings.getLpPipeline().subjectBatchSize(); this.BATCH_SIZE = creditTrackerSettings.getLpPipeline().subjectBatchSize();
this.daysToUpdateFids = creditTrackerSettings.getDaysToUpdateFids() != null ? creditTrackerSettings.getDaysToUpdateFids() : 1; this.daysToUpdateFids = creditTrackerSettings.getDaysToUpdateLpFids() != null ? creditTrackerSettings.getDaysToUpdateLpFids() : 1;
} }
private static class PipelineStarterException extends SignalClientException { private static class PipelineStarterException extends SignalClientException {

View file

@ -80,7 +80,7 @@ public class NpFidUpdater implements Handler<SubjectPayload, VoidPayload> {
} }
if (!changingFids.isEmpty()) { if (!changingFids.isEmpty()) {
List<Long> packageIds = packageRecordsService.selectUniquePackageIds(changingFids); List<Long> packageIds = packageRecordsService.selectNpUniquePackageIds(changingFids);
if (!packageIds.isEmpty()) { if (!packageIds.isEmpty()) {
userPackageService.setFidChanged(packageIds); userPackageService.setFidChanged(packageIds);
} }

View file

@ -44,7 +44,7 @@ public class NpKickRecalcPipeLine implements IPipeLineStarter<SubjectPayload> {
this.packageRecordsService = packageRecordsService; this.packageRecordsService = packageRecordsService;
this.creditTrackerSettings = creditTrackerSettings; this.creditTrackerSettings = creditTrackerSettings;
this.BATCH_SIZE = creditTrackerSettings.getPipeline().subjectBatchSize(); this.BATCH_SIZE = creditTrackerSettings.getPipeline().subjectBatchSize();
this.daysToUpdateFids = creditTrackerSettings.getDaysToUpdateFids() != null ? creditTrackerSettings.getDaysToUpdateFids() : 1; this.daysToUpdateFids = creditTrackerSettings.getDaysToUpdateNpFids() != null ? creditTrackerSettings.getDaysToUpdateNpFids() : 1;
} }
private static class PipelineStarterException extends SignalClientException { private static class PipelineStarterException extends SignalClientException {

View file

@ -113,7 +113,7 @@
flag_90_12 flag_90_12
FROM json_to_recordset(#{json}::json) AS src( FROM json_to_recordset(#{json}::json) AS src(
inn bigint, inn bigint,
reg_num text, reg_num bigint,
flag_90_12 smallint flag_90_12 smallint
) )
ORDER BY inn, reg_num), ORDER BY inn, reg_num),
@ -128,7 +128,7 @@
calculated_at = now() calculated_at = now()
FROM json_to_recordset(#{json}::json) AS src( FROM json_to_recordset(#{json}::json) AS src(
inn bigint, inn bigint,
reg_num text, reg_num bigint,
flag_90_12 smallint flag_90_12 smallint
) )
WHERE dst.inn = src.inn WHERE dst.inn = src.inn
@ -143,7 +143,7 @@
last_rejected_date last_rejected_date
FROM json_to_recordset(#{json}::json) AS src( FROM json_to_recordset(#{json}::json) AS src(
inn bigint, inn bigint,
reg_num text, reg_num bigint,
last_rejected_date timestamp last_rejected_date timestamp
) )
ORDER BY inn, reg_num), ORDER BY inn, reg_num),
@ -157,7 +157,7 @@
SET last_rejected_date = src.last_rejected_date SET last_rejected_date = src.last_rejected_date
FROM json_to_recordset(#{json}::json) AS src( FROM json_to_recordset(#{json}::json) AS src(
inn bigint, inn bigint,
reg_num text, reg_num bigint,
last_rejected_date timestamp last_rejected_date timestamp
) )
WHERE dst.inn = src.inn WHERE dst.inn = src.inn
@ -180,4 +180,60 @@
order by id order by id
limit #{batchSize} limit #{batchSize}
</select> </select>
<update id="updateSearchedAt" parameterType="string">
update signals_legals_map dst
set searched_at = now()
FROM json_to_recordset(#{json}::json) AS src(
inn bigint,
reg_num bigint)
where dst.inn = src.inn AND dst.reg_num = src.reg_num
</update>
<update id="resetFidAndFlagsByInnAndRegNum" parameterType="string">
update signals_legals_map dst
set searched_at = now(),
fid = null,
flag_90_12 = null,
calculated_at = null,
last_rejected_date = null
FROM json_to_recordset(#{json}::json) AS src(
inn bigint,
reg_num bigint)
where dst.inn = src.inn AND dst.reg_num = src.reg_num
</update>
<update id="updateFidCachedData" parameterType="string">
update signals_legals_map dst
set searched_at = now(),
fid = src.fid,
flag_90_12 = null,
calculated_at = null,
last_rejected_date = null
FROM json_to_recordset(#{json}::json) AS src(
inn bigint,
reg_num bigint,
fid int)
where dst.inn = src.inn AND dst.reg_num = src.reg_num
</update>
<delete id="deleteUnusedMaps">
DELETE
FROM signals_legals_map slm
WHERE NOT EXISTS (SELECT 1
FROM signals_legal_fid_map slr
JOIN signals_user_packages sup
ON sup.id = slr.package_id
WHERE slr.inn = slm.inn
AND slr.reg_num = slm.reg_num
AND sup.status IN
(
'ON_MONITORING',
'PROCESSING',
'PROCESSING_REPEATED'
))
</delete>
</mapper> </mapper>

View file

@ -53,6 +53,19 @@
where spr.phone_l in (select * from src) and sup.status = 'ON_MONITORING' where spr.phone_l in (select * from src) and sup.status = 'ON_MONITORING'
</select> </select>
<select id="uniquePackageIdByInnAndRegNum" resultType="java.lang.Long">
WITH src AS (SELECT inn, regNum
FROM json_to_recordset(#{json}::json) AS t(
inn bigint,
regNum bigint
))
select distinct (package_id)
FROM signals_legal_fid_map slfm
JOIN signals_user_packages sup ON sup.id = slfm.package_id
WHERE (slfm.inn, slfm.reg_num) IN (SELECT inn, regNum FROM src)
AND sup.status = 'ON_MONITORING'
</select>
<select id="selectUserPackagePhoneData" resultType="ru.nbch.credit_tracker.model.UserPackagePhoneData"> <select id="selectUserPackagePhoneData" resultType="ru.nbch.credit_tracker.model.UserPackagePhoneData">
WITH src AS (SELECT phone WITH src AS (SELECT phone
FROM json_to_recordset(#{json}::json) AS t( FROM json_to_recordset(#{json}::json) AS t(