From d3fc6ce7e8413d28960a01e6b29cc0ed8d661a52 Mon Sep 17 00:00:00 2001 From: Mikhail Trofimov Date: Fri, 3 Jul 2026 15:59:31 +0300 Subject: [PATCH] Added enew env for lp batch size saving --- credit-tracker.properties | 10 ++++++---- docker_compose/credit_tracker_config.env | 3 ++- docker_compose/docker-compose.yml | 3 ++- .../config/settings/CreditTrackerSettings.java | 3 ++- .../nbch/credit_tracker/service/task/impl/Saver.java | 10 ++++++---- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/credit-tracker.properties b/credit-tracker.properties index 0ec3758..36d5e9d 100644 --- a/credit-tracker.properties +++ b/credit-tracker.properties @@ -102,7 +102,9 @@ ct.encoding=windows-1251 #----------------------------Pipeline settings----------------------------# #batch size for saving in phone_fid_map -ct.saver-subject-batch-size=${SAVER_SUBJECT_BATCH_SIZE:15000} +ct.npsaver-subject-batch-size=${NP_SAVER_SUBJECT_BATCH_SIZE:15000} +#batch size for saving in signals_legal_fid_map +ct.lp-saver-subject-batch-size=${LP_SAVER_SUBJECT_BATCH_SIZE:15000} #limit on select phone_fid_map ct.np-pipeline.subject-batch-size=${NP_PIPELINE_SUBJECT_BATCH_SIZE:10000} #limit on select signals_legal_map @@ -145,9 +147,9 @@ ct.np-pipeline.error-attempt-count=${NP_PIPELINE_ERROR_ATTEMPT_COUNT:5} ct.np-pipeline.error-attempt-delay=${NP_PIPELINE_ERROR_ATTEMPT_DELAY:2S} ct.np-pipeline.retryable-exceptions=${NP_PIPELINE_RETRYABLE_EXCEPTIONS:org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException} -ct.LP-pipeline.error-attempt-count=${LP_PIPELINE_ERROR_ATTEMPT_COUNT:5} -ct.LP-pipeline.error-attempt-delay=${LP_PIPELINE_ERROR_ATTEMPT_DELAY:2S} -ct.LP-pipeline.retryable-exceptions=${LP_PIPELINE_RETRYABLE_EXCEPTIONS:org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException} +ct.lp-pipeline.error-attempt-count=${LP_PIPELINE_ERROR_ATTEMPT_COUNT:5} +ct.lp-pipeline.error-attempt-delay=${LP_PIPELINE_ERROR_ATTEMPT_DELAY:2S} +ct.lp-pipeline.retryable-exceptions=${LP_PIPELINE_RETRYABLE_EXCEPTIONS:org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException} #Pipeline global settings ct.np-pipeline.global-timeout=${NP_PIPELINE_GLOBAL_TIMEOUT:2H} diff --git a/docker_compose/credit_tracker_config.env b/docker_compose/credit_tracker_config.env index c9399c7..4690982 100644 --- a/docker_compose/credit_tracker_config.env +++ b/docker_compose/credit_tracker_config.env @@ -103,11 +103,11 @@ UPDATE_FIDS_CRON_EXPRESSION=0 10 12 * * * DAYS_TO_UPDATE_FIDS=30 # общие настройки -SAVER_SUBJECT_BATCH_SIZE=15000 FLAG_UPDATER_CONCURRENCY=10 FLAG_UPDATER_QUEUE_CAPACITY=100 # Физ лица +NP_SAVER_SUBJECT_BATCH_SIZE=15000 NP_PIPELINE_SUBJECT_BATCH_SIZE=10000 NP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=20 NP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=20 @@ -121,6 +121,7 @@ NP_PIPELINE_GLOBAL_TIMEOUT=2H NP_PIPELINE_PARALLEL_REQUESTS=3 # Юр. Лица +LP_SAVER_SUBJECT_BATCH_SIZE=15000 LP_PIPELINE_SUBJECT_BATCH_SIZE=10000 LP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=20 LP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=20 diff --git a/docker_compose/docker-compose.yml b/docker_compose/docker-compose.yml index c6db10d..15d814f 100644 --- a/docker_compose/docker-compose.yml +++ b/docker_compose/docker-compose.yml @@ -87,9 +87,9 @@ services: - UPDATE_FIDS_CRON_EXPRESSION=${UPDATE_FIDS_CRON_EXPRESSION} - DAYS_TO_UPDATE_FIDS=${DAYS_TO_UPDATE_FIDS} - DAYS_TO_UPDATE_FLAGS=${DAYS_TO_UPDATE_FLAGS} - - SAVER_SUBJECT_BATCH_SIZE=${SAVER_SUBJECT_BATCH_SIZE} - FLAG_UPDATER_CONCURRENCY=${FLAG_UPDATER_CONCURRENCY} - FLAG_UPDATER_QUEUE_CAPACITY=${FLAG_UPDATER_QUEUE_CAPACITY} + - NP_SAVER_SUBJECT_BATCH_SIZE=${NP_SAVER_SUBJECT_BATCH_SIZE} - NP_PIPELINE_SUBJECT_BATCH_SIZE=${NP_PIPELINE_SUBJECT_BATCH_SIZE} - NP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=${NP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY} - NP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=${NP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY} @@ -101,6 +101,7 @@ services: - NP_PIPELINE_RETRYABLE_EXCEPTIONS=${NP_PIPELINE_RETRYABLE_EXCEPTIONS} - NP_PIPELINE_GLOBAL_TIMEOUT=${NP_PIPELINE_GLOBAL_TIMEOUT} - NP_PIPELINE_PARALLEL_REQUESTS=${NP_PIPELINE_PARALLEL_REQUESTS} + - LP_SAVER_SUBJECT_BATCH_SIZE=${LP_SAVER_SUBJECT_BATCH_SIZE} - LP_PIPELINE_SUBJECT_BATCH_SIZE=${LP_PIPELINE_SUBJECT_BATCH_SIZE} - LP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=${LP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY} - LP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=${LP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY} diff --git a/src/main/java/ru/nbch/credit_tracker/config/settings/CreditTrackerSettings.java b/src/main/java/ru/nbch/credit_tracker/config/settings/CreditTrackerSettings.java index d12b900..04e5bc8 100644 --- a/src/main/java/ru/nbch/credit_tracker/config/settings/CreditTrackerSettings.java +++ b/src/main/java/ru/nbch/credit_tracker/config/settings/CreditTrackerSettings.java @@ -42,5 +42,6 @@ public class CreditTrackerSettings { private Long daysToUpdateNpFids; private Long daysToUpdateLpFids; private String encoding; - private Integer saverSubjectBatchSize; + private Integer npSaverSubjectBatchSize; + private Integer lpSaverSubjectBatchSize; } diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/impl/Saver.java b/src/main/java/ru/nbch/credit_tracker/service/task/impl/Saver.java index 7735054..b709750 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/impl/Saver.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/impl/Saver.java @@ -27,7 +27,8 @@ public class Saver { private final PhoneService phoneService; private final LegalsMapService legalsMapService; private final UserPackageMapper userPackageMapper; - private final int BATCH_SIZE; + private final int NP_BATCH_SIZE; + private final int LP_BATCH_SIZE; public Saver(StaxXmlProcessor xmlProcessor, @@ -39,7 +40,8 @@ public class Saver { this.phoneService = phoneService; this.legalsMapService = legalsMapService; this.userPackageMapper = userPackageMapper; - this.BATCH_SIZE = creditTrackerSettings.getSaverSubjectBatchSize(); + this.NP_BATCH_SIZE = creditTrackerSettings.getNpSaverSubjectBatchSize(); + this.LP_BATCH_SIZE = creditTrackerSettings.getLpSaverSubjectBatchSize(); } public void readAndSaveNpPackage(Long packageId, Path pkgPath) { @@ -59,7 +61,7 @@ public class Saver { } }, request -> request.getSubjects().getSubject().isEmpty(), - BATCH_SIZE, "/MonitoringRequest/Subjects/Subject" + NP_BATCH_SIZE, "/MonitoringRequest/Subjects/Subject" ); } catch (IOException e) { log.error("{}", ExceptionUtils.getStackTrace(e)); @@ -87,7 +89,7 @@ public class Saver { } }, request -> request.getSubjects().getSubject().isEmpty(), - BATCH_SIZE, "/MonitoringRequest/Subjects/Subject" + LP_BATCH_SIZE, "/MonitoringRequest/Subjects/Subject" ); } catch (IOException e) { log.error("{}", ExceptionUtils.getStackTrace(e));