From a90d76d209f8a01ee0ff04d426d0a7d11aa9c26a Mon Sep 17 00:00:00 2001 From: etreschenkov Date: Tue, 20 Jan 2026 10:27:13 +0300 Subject: [PATCH 01/10] http://jira.mfd.msk:8088/browse/BKI-3241 --- .../service/UpdatingFidAndPdService.java | 35 +++++++++++-------- .../task/PipelineRecalcPDOrchestrator.java | 3 +- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFidAndPdService.java b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFidAndPdService.java index 5193514..47aae57 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/UpdatingFidAndPdService.java +++ b/src/main/java/ru/nbch/credit_tracker/service/UpdatingFidAndPdService.java @@ -49,23 +49,28 @@ public class UpdatingFidAndPdService { Pipeline pipeline = pipelinePackageManager.buildRecalculationPD(new SetupParam(null, null, PipelineMode.RECALC_PERSONAL_DATA)); CompletableFuture future = pipelineRecalcPDOrchestrator.startOrReject(pipeline); future.whenComplete((o, throwable) -> { - if (throwable != null) { - log.error(throwable.getMessage(), throwable); - return; - } - - List packagesRepeated = userPackageMapper.findForRepeatedPackages(); - packagesRepeated.forEach(userPackage -> { - Long packageId = userPackage.getId(); - SetupParam setupParam = new SetupParam(packageId, null, PipelineMode.DEFAULT); - try { - Pipeline makeXmlByCache = pipelinePackageManager.build(setupParam); - pipelinePackageOrchestrator.startOrDelay(makeXmlByCache, packageId); - } catch (Exception e) { - log.error("Failed to build pipeline", e); + try { + if (throwable != null) { + log.error(throwable.getMessage(), throwable); + return; } - }); + List packagesRepeated = userPackageMapper.findForRepeatedPackages(); + packagesRepeated.forEach(userPackage -> { + Long packageId = userPackage.getId(); + SetupParam setupParam = new SetupParam(packageId, null, PipelineMode.DEFAULT); + try { + Pipeline makeXmlByCache = pipelinePackageManager.build(setupParam); + pipelinePackageOrchestrator.startOrDelay(makeXmlByCache, packageId); + } catch (Exception e) { + log.error("Failed to build pipeline", e); + } + }); + } finally { + log.info("pipeline finished"); + pipeline.close(); + pipelineRecalcPDOrchestrator.release(); + } }); } catch (Exception e) { log.error("Failed to build pipeline", e); diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/PipelineRecalcPDOrchestrator.java b/src/main/java/ru/nbch/credit_tracker/service/task/PipelineRecalcPDOrchestrator.java index 0a92801..f024ed6 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/PipelineRecalcPDOrchestrator.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/PipelineRecalcPDOrchestrator.java @@ -54,10 +54,11 @@ public class PipelineRecalcPDOrchestrator { .orTimeout(creditTrackerSettings.getPipeline().globalTimeout().toMillis(), TimeUnit.MILLISECONDS); } - private void release() { + public void release() { log.debug("checking for delayed pipelines..."); synchronized (lock) { semaphore.release(); + log.info("semaphore amount: {}", semaphore.availablePermits()); } } } From 4994af50bd9b6e7b0954814814abd9a669489218 Mon Sep 17 00:00:00 2001 From: etreschenkov Date: Tue, 20 Jan 2026 10:27:47 +0300 Subject: [PATCH 02/10] http://jira.mfd.msk:8088/browse/BKI-3241 --- credit-tracker.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/credit-tracker.properties b/credit-tracker.properties index 805c183..ed5aea6 100644 --- a/credit-tracker.properties +++ b/credit-tracker.properties @@ -68,7 +68,7 @@ ct.monitoring-interval=${MONITORING_INTERVAL:15} # days ct.update-flags-cron-expression=${UPDATE_FLAGS_CRON_EXPRESSION:0 25 18 * * *} ct.days-to-update-flags=${DAYS_TO_UPDATE_FLAGS:30} -ct.update-fid-cron-expression=${UPDATE_FLAGS_CRON_EXPRESSION:0 10 12 * * *} +ct.update-fid-cron-expression=${UPDATE_FIDS_CRON_EXPRESSION:0 10 12 * * *} ct.days-to-update-fids=${DAYS_TO_UPDATE_FIDS:30} ct.db-version=${DB_VERSION:1234} From c32d4304833920ca4b803f4eaf8d99a053f849c6 Mon Sep 17 00:00:00 2001 From: etreschenkov Date: Tue, 20 Jan 2026 13:36:48 +0300 Subject: [PATCH 03/10] http://jira.mfd.msk:8088/browse/BKI-3241 --- .../service/task/impl/FidNameUpdater.java | 18 ++++++++++-------- .../mapper/signal/PackageRecordsMapper.xml | 5 ++++- .../mybatis/mapper/signal/PhonesMapMapper.xml | 3 ++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/impl/FidNameUpdater.java b/src/main/java/ru/nbch/credit_tracker/service/task/impl/FidNameUpdater.java index f778c0c..cb2f859 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/impl/FidNameUpdater.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/impl/FidNameUpdater.java @@ -49,15 +49,17 @@ public class FidNameUpdater implements Handler { List notFoundFidPhones = new ArrayList<>(); subjects.forEach(subjectProfile -> { Name name = nameByPhone.get(CTUtil.normalizePhoneNumber(subjectProfile.getPhone())); - if (subjectProfile.getFid() == null && name == null) { + if (name != null) { + if (subjectProfile.getFid() == null || !subjectProfile.getFid().equals(name.getFid())) { + subjectProfile.setFid(name.getFid()); + subjectProfile.getNameData().setFirstName(name.getFirst()); + subjectProfile.getNameData().setLastName(name.getLast()); + subjectProfile.getNameData().setMiddleName(name.getMiddle()); + subjectProfile.getNameData().setBirthDate(name.getBirthDate()); + changingFidPhones.add(subjectProfile); + } + } else if (subjectProfile.getFid() != null) { notFoundFidPhones.add(subjectProfile.getPhone()); - } else if (subjectProfile.getFid() == null || !subjectProfile.getFid().equals(name.getFid())) { - subjectProfile.setFid(name.getFid()); - subjectProfile.getNameData().setFirstName(name.getFirst()); - subjectProfile.getNameData().setLastName(name.getLast()); - subjectProfile.getNameData().setMiddleName(name.getMiddle()); - subjectProfile.getNameData().setBirthDate(name.getBirthDate()); - changingFidPhones.add(subjectProfile); } }); List changingFids = new ArrayList<>(); diff --git a/src/main/resources/mybatis/mapper/signal/PackageRecordsMapper.xml b/src/main/resources/mybatis/mapper/signal/PackageRecordsMapper.xml index e113bbd..4b19495 100644 --- a/src/main/resources/mybatis/mapper/signal/PackageRecordsMapper.xml +++ b/src/main/resources/mybatis/mapper/signal/PackageRecordsMapper.xml @@ -28,7 +28,10 @@ FROM json_to_recordset(#{json}::json) AS t( phone bigint )) - select distinct (package_id) from signals_packages_records where phone_l in (select * from src) + select distinct (package_id) + from signals_packages_records spr + join signals_user_packages sup on sup.id = spr.package_id + where spr.phone_l in (select * from src) and sup.status = 'ON_MONITORING' + + + @@ -29,7 +29,7 @@ phone bigint )) select distinct (package_id) - from signals_packages_records spr + FROM signals_phone_fid_map spr join signals_user_packages sup on sup.id = spr.package_id where spr.phone_l in (select * from src) and sup.status = 'ON_MONITORING' @@ -45,7 +45,7 @@ package_id bigint )) INSERT - INTO signals_packages_records + INTO signals_phone_fid_map (subject_id, phone, phone_l, package_id) SELECT subject_id, phone, @@ -57,7 +57,7 @@ - update signals_packages_records dst + update signals_phone_fid_map dst set error_text = src.error_text FROM json_to_recordset(#{json}::json) AS src( id bigint, @@ -68,7 +68,7 @@ DELETE - FROM signals_packages_records + FROM signals_phone_fid_map WHERE package_id = #{packageId} @@ -78,7 +78,7 @@ subject_id, phone, error_text - FROM signals_packages_records + FROM signals_phone_fid_map WHERE package_id = #{packageId} ORDER BY id LIMIT #{limit} OFFSET #{offset} diff --git a/src/main/resources/mybatis/mapper/signal/PhonesMapMapper.xml b/src/main/resources/mybatis/mapper/signal/PhonesMapMapper.xml index b9c409f..0abefbe 100644 --- a/src/main/resources/mybatis/mapper/signal/PhonesMapMapper.xml +++ b/src/main/resources/mybatis/mapper/signal/PhonesMapMapper.xml @@ -213,7 +213,7 @@ DELETE FROM signals_phones_map spm WHERE NOT EXISTS (SELECT 1 - FROM signals_packages_records spr + FROM signals_phone_fid_map spr JOIN signals_user_packages sup ON sup.id = spr.package_id WHERE spr.phone_l = spm.phone diff --git a/src/main/resources/sql/credit_tracker_db.ddl b/src/main/resources/sql/credit_tracker_db.ddl index f79c2f4..efa00bc 100644 --- a/src/main/resources/sql/credit_tracker_db.ddl +++ b/src/main/resources/sql/credit_tracker_db.ddl @@ -1,43 +1,94 @@ --- Create signals_user_packages --- Хранит основную информацию о каждом пакете мониторинга, установленном клиентом -CREATE TABLE signals_user_packages ( - id SERIAL PRIMARY KEY, -- внутренний идентификатор в таблице - membercode VARCHAR(255), -- логин клиента - package_name VARCHAR(255), -- имя пакета - created_at TIMESTAMP, -- дата и время первичной регистрации пакета - status VARCHAR(255), -- статус пакета: ON_MONITORING, REJECTED, PROCESSING, DELETED, ERROR - deactivated_at TIMESTAMP, -- дата и время удаления пакета - last_downloaded_report VARCHAR(255) -- имя последнего успешно загруженного отчёта из оф. сервиса по пакету +create table signals_user_packages +( + id bigserial + primary key, + membercode varchar(255), + package_name varchar(255), + created_at timestamp, + status varchar(255), + deactivated_at timestamp, + last_downloaded_report varchar(255), + monitoring_at timestamp, + last_searched_at timestamp, + signal_types varchar(255), + reject_status varchar(255), + external_package_name varchar(255), + fid_changed smallint ); --- Create signals_phone_fid_map -CREATE TABLE signals_phone_fid_map ( - id SERIAL PRIMARY KEY, -- внутренний идентификатор в таблице - package_id INTEGER NOT NULL, -- идентификатор пакета - subject_id VARCHAR(255), -- идентификатор субъекта из XML - phone VARCHAR(255), -- номер телефона - fid VARCHAR(255), -- найденный FID или NULL - flag_debt5000 SMALLINT, -- 1, если есть активная просрочка > 5000 руб., иначе 0 - flag_90_12 SMALLINT, -- 1, если по субъекту когда-либо была просрочка 90+ дней за последний год, иначе – 0 - calculated_at TIMESTAMP, -- дата последнего расчёта флагов - error_text TEXT, - CONSTRAINT fk_signals_user_packages - FOREIGN KEY (package_id) - REFERENCES signals_user_packages(id) +alter table signals_user_packages + owner to credit_tracker_user; + +create table signals_hits +( + id bigserial + primary key, + package_id bigint not null + constraint fk_signals_user_packages + references signals_user_packages, + subject_id varchar(255), + signal_type integer, + response_sent_at timestamp, + report_file_name varchar(255), + phone bigint ); --- Create signals_hits -CREATE TABLE signals_hits ( - id SERIAL PRIMARY KEY, -- внутренний идентификатор в таблице - package_id INTEGER NOT NULL, -- идентификатор пакета - subject_id VARCHAR(255), -- идентификатор субъекта, по которому сработал сигнал - signal_type INTEGER, -- тип сигнала - response_sent_at TIMESTAMP, -- дата и время формирования отчета с данным сигналом - report_file_name VARCHAR(255), -- имя XML-файла отчёта, в который вошел данный сигнал - CONSTRAINT fk_signals_user_packages - FOREIGN KEY (package_id) - REFERENCES signals_user_packages(id) +alter table signals_hits + owner to credit_tracker_user; + +create index idx_signals_hits_response_sent_at + on signals_hits (response_sent_at); + +create table signals_phone_fid_map +( + id bigserial + primary key, + package_id bigint not null + constraint fk_signals_user_packages + references signals_user_packages, + subject_id varchar(255), + phone varchar(255), + fid varchar(255), + flag_debt5000 smallint, + flag_90_12 smallint, + calculated_at timestamp, + error_text text, + phone_l bigint ); --- Create index on response_sent_at -CREATE INDEX idx_signals_hits_response_sent_at ON signals_hits(response_sent_at); \ No newline at end of file +alter table signals_phone_fid_map + owner to credit_tracker_user; + +create index idx_signals_phone_fid_map_calculated_at + on signals_phone_fid_map (calculated_at); + +create table app_version +( + db_version varchar(12) +); + +alter table app_version + owner to credit_tracker_user; + +create table signals_phones_map +( + id bigserial + primary key, + fid integer, + phone bigint + unique, + name_1 varchar(255), + first varchar(255), + middle varchar(255), + birth_dt timestamp, + ser_num varchar(255), + id_type varchar(255), + flag_debt5000 smallint, + flag_90_12 smallint, + searched_at timestamp, + calculated_at timestamp +); + +alter table signals_phones_map + owner to credit_tracker_user; + diff --git a/src/main/resources/sql/v2.21.ddl b/src/main/resources/sql/v2.21.ddl index 38ad0c7..3b73e0f 100644 --- a/src/main/resources/sql/v2.21.ddl +++ b/src/main/resources/sql/v2.21.ddl @@ -1,8 +1,3 @@ -CREATE TABLE app_version -( - DB_VERSION varchar(12) -); - alter table signals_user_packages add LAST_SEARCHED_AT timestamp; alter table signals_user_packages @@ -13,13 +8,17 @@ alter table signals_user_packages add EXTERNAL_PACKAGE_NAME varchar(255); alter table signals_user_packages add FID_CHANGED smallint; -alter table signals_packages_records - add phone_l bigint; - alter table signals_hits add PHONE bigint; +alter table signals_phone_fid_map + add phone_l bigint; + +CREATE TABLE app_version +( + DB_VERSION varchar(12) +); CREATE TABLE signals_phones_map ( @@ -41,13 +40,4 @@ CREATE TABLE signals_phones_map ALTER TABLE signals_phones_map ADD CONSTRAINT signals_phones_map_phone_key UNIQUE (PHONE); - - -alter table signals_phone_fid_map rename to signals_packages_records; -alter table signals_packages_records drop column fid; -alter table signals_packages_records drop column flag_debt5000; -alter table signals_packages_records drop column flag_90_12; -alter table signals_packages_records drop column calculated_at; -alter table signals_packages_records drop column rec_uuid; -alter table signals_packages_records drop column searched_at; --- todo alter indexes and other constrains +update signals_phone_fid_map set phone_l = phone::BIGINT where phone is not null; \ No newline at end of file