Get rid of InquiryDateDeserializer; Changed ack mode

LP: added lp kick pipelines with data loading

.gitignore

@@ -31,3 +31,4 @@ build/
 
 ### VS Code ###
 .vscode/
+tmp/

Dockerfile

@@ -1,4 +1,4 @@
-FROM eclipse-temurin:17-jre-alpine
+FROM eclipse-temurin:21-jre-alpine
 MAINTAINER ru.nbch
 
 VOLUME /opt/app/logs/app
@@ -8,7 +8,8 @@ VOLUME /opt/app/reports
 
 WORKDIR /opt/app
 COPY maven/elastic-apm-agent-1.45.0.jar /opt/app/elastic-apm-agent.jar
-COPY maven/credit-tracker.jar /opt/app/credit-tracker.jar
+COPY maven/credit-tracker-main.jar /opt/app/credit-tracker-main.jar
+COPY maven/credit-tracker-monitoring.jar /opt/app/credit-tracker-monitoring.jar
 COPY maven/start-credit-tracker.sh /opt/app/start-credit-tracker.sh
 COPY maven/credit-tracker.properties /opt/app/credit-tracker.properties
 COPY maven/monitoring-request.xsd /opt/app/monitoring-request.xsd
@@ -20,4 +21,3 @@ RUN apk add --no-cache ca-certificates \
 RUN apk add gcompat bash
 
 RUN chmod +x /opt/app/start-credit-tracker.sh
-ENTRYPOINT ["/opt/app/start-credit-tracker.sh"]

credit-tracker.properties

@@ -8,10 +8,10 @@ server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b
 # datasources
 # indic
 ct.indic-data-source-settings.driver=com.ibm.db2.jcc.DB2Driver
-ct.indic-data-source-settings.url=${DB_URL_INDIC:jdbc:db2://10.230.227.101:2668/cprosd22:currentSchema=INDIC;}
-ct.indic-data-source-settings.username=${USERNAME_INDIC:tester}
-ct.indic-data-source-settings.password=${PASSWORD_INDIC:tester}
-ct.indic-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_INDIC:10}
+ct.indic-data-source-settings.url=${DB_URL_INDIC:jdbc:db2://10.230.227.102:2668/cprosd22:currentSchema=INDIC;}
+ct.indic-data-source-settings.username=${USERNAME_INDIC:ros9appl}
+ct.indic-data-source-settings.password=${PASSWORD_INDIC:R0s9Appl}
+ct.indic-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_INDIC:30}
 ct.indic-data-source-settings.minimum-idle=${MINIMUM_IDLE_INDIC:10}
 ct.indic-data-source-settings.connection-timeout=${CONNECTION_TIMEOUT_INDIC:10000}
 ct.indic-data-source-settings.validation-timeout=${VALIDATION_TIMEOUT_INDIC:5000}
@@ -22,10 +22,10 @@ ct.indic-data-source-settings.leak-detection-threshold=${LEAK_DETECTION_THRESHOL
 
 # signal
 ct.signal-data-source-settings.driver=org.postgresql.Driver
-ct.signal-data-source-settings.url=${DB_URL_SIGNAL:jdbc:postgresql://10.200.200.133:5432/credit_tracker}
+ct.signal-data-source-settings.url=${DB_URL_SIGNAL:jdbc:postgresql://10.200.200.133:5432/credit_tracker?currentSchema=test_3}
 ct.signal-data-source-settings.username=${USERNAME_SIGNAL:credit_tracker_user}
 ct.signal-data-source-settings.password=${PASSWORD_SIGNAL:-}
-ct.signal-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SIGNAL:10}
+ct.signal-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SIGNAL:40}
 ct.signal-data-source-settings.minimum-idle=${MINIMUM_IDLE_SIGNAL:10}
 ct.signal-data-source-settings.connection-timeout=${CONNECTION_TIMEOUT_SIGNAL:10000}
 ct.signal-data-source-settings.validation-timeout=${VALIDATION_TIMEOUT_SIGNAL:5000}
@@ -39,7 +39,7 @@ ct.scoring-data-source-settings.driver=com.ibm.db2.jcc.DB2Driver
 ct.scoring-data-source-settings.url=${DB_URL_SCORING:jdbc:db2://10.230.227.101:2668/cprosd22:currentSchema=TEST_2_INDIC;}
 ct.scoring-data-source-settings.username=${USERNAME_SCORING:tester}
 ct.scoring-data-source-settings.password=${PASSWORD_SCORING:tester}
-ct.scoring-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SCORING:10}
+ct.scoring-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SCORING:30}
 ct.scoring-data-source-settings.minimum-idle=${MINIMUM_IDLE_SCORING:10}
 ct.scoring-data-source-settings.connection-timeout=${CONNECTION_TIMEOUT_SCORING:10000}
 ct.scoring-data-source-settings.validation-timeout=${VALIDATION_TIMEOUT_SCORING:5000}
@@ -48,6 +48,20 @@ ct.scoring-data-source-settings.max-lifetime=${MAX_LIFETIME_SCORING:1800000}
 ct.scoring-data-source-settings.keepalive-time=${KEEPALIVE_TIME_SCORING:0}
 ct.scoring-data-source-settings.leak-detection-threshold=${LEAK_DETECTION_THRESHOLD_SCORING:0}
 
+# nbch
+ct.nbch-data-source-settings.driver=com.ibm.db2.jcc.DB2Driver
+ct.nbch-data-source-settings.url=${DB_URL_NBCH:jdbc:db2://10.230.227.102:2668/cprosd22:currentSchema=NBCH;}
+ct.nbch-data-source-settings.username=${USERNAME_NBCH:ros9appl}
+ct.nbch-data-source-settings.password=${PASSWORD_NBCH:R0s9Appl}
+ct.nbch-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_NBCH:30}
+ct.nbch-data-source-settings.minimum-idle=${MINIMUM_IDLE_NBCH:10}
+ct.nbch-data-source-settings.connection-timeout=${CONNECTION_TIMEOUT_NBCH:10000}
+ct.nbch-data-source-settings.validation-timeout=${VALIDATION_TIMEOUT_NBCH:5000}
+ct.nbch-data-source-settings.idle-timeout=${IDLE_TIMEOUT_NBCH:600000}
+ct.nbch-data-source-settings.max-lifetime=${MAX_LIFETIME_NBCH:1800000}
+ct.nbch-data-source-settings.keepalive-time=${KEEPALIVE_TIME_NBCH:0}
+ct.nbch-data-source-settings.leak-detection-threshold=${LEAK_DETECTION_THRESHOLD_NBCH:0}
+
 ct.max_xml_file_size=${ACCEPTED_XML_SIZE:1024}
 spring.servlet.multipart.max-file-size=${ACCEPTED_ARCHIVE_SIZE:2048MB}
 spring.servlet.multipart.max-request-size=${ACCEPTED_ARCHIVE_SIZE:2048MB}
@@ -56,17 +70,90 @@ spring.servlet.multipart.max-request-size=${ACCEPTED_ARCHIVE_SIZE:2048MB}
 ct.xsd-validation.monitoring-request-path=monitoring-request.xsd
 ct.xsd-validation.signal-package-path=signal-package.xsd
 
-ct.signals-root-uri=${SIGNALS_ROOT_URI:http://10.230.227.102:11094/signalRUalfa/}
-ct.signals-token=${SIGNALS_TOKEN:gttcHxwcGojoAvZK}
-ct.signals-user=${SIGNALS_USER:0001ZZ000001}
-
 ct.load-dir=/opt/app/reports
 #ct.load-dir=D:\\folder_name\\signals
+ct.tmp.dir=D:\\folder_name\\signals\\tmp
+ct.target.dir=D:\\folder_name\\signals\\tmpt
 
 # min
-ct.monitoring-interval=${MONITORING_INTERVAL:15}
+ct.monitoring-interval=${MONITORING_INTERVAL:5}
 # days
-ct.update-flags-interval=${UPDATE_FLAGS_INTERVAL:1}
+ct.update-flags-cron-expression=${UPDATE_FLAGS_CRON_EXPRESSION:0 25 18 * * *}
+ct.new-flags-cron-expression=${NEW_FLAGS_CRON_EXPRESSION:0 25 18 * * *}
 ct.days-to-update-flags=${DAYS_TO_UPDATE_FLAGS:30}
+ct.update-np-fid-cron-expression=${UPDATE_FIDS_CRON_EXPRESSION:0 10 12 * * *}
+ct.update-lp-fid-cron-expression=${UPDATE_FIDS_CRON_EXPRESSION:0 10 12 * * *}
+ct.days-to-update-np-fids=${DAYS_TO_UPDATE_FIDS:30}
+ct.days-to-update-lp-fids=${DAYS_TO_UPDATE_FIDS:30}
+
+#kafka
+ct.kafka-consumer-settings.topic=${CONSUMER_TOPIC:my-topic}
+ct.kafka-consumer-settings.concurency=${CONSUMER_CONCURENCY:8}
+ct.kafka-consumer-settings.bootstrap-servers=${CONSUMER_BOOTSTRAP_SERVERS:localhost:29092}
+ct.kafka-consumer-settings.group-id=${CONSUMER_GROUP_ID:credit-tracker}
+ct.kafka-consumer-settings.auto-offset-reset=${CONSUMER_AUTO_OFFSET_RESET:latest}
+ct.kafka-consumer-settings.max-poll-records=${CONSUMER_MAX_POLL_RECORDS:500}
+ct.kafka-consumer-settings.max-poll-interval-ms=${CONSUMER_MAX_POLL_INTERVAL_MS:300000}
+
+ct.db-version=${DB_VERSION:1234}
 
 ct.encoding=windows-1251
+
+
+#----------------------------Pipeline settings----------------------------#
+#batch size for saving in phone_fid_map
+ct.npsaver-subject-batch-size=${NP_SAVER_SUBJECT_BATCH_SIZE:15000}
+#batch size for saving in signals_legal_fid_map
+ct.lp-saver-subject-batch-size=${LP_SAVER_SUBJECT_BATCH_SIZE:15000}
+#limit on select phone_fid_map
+ct.np-pipeline.subject-batch-size=${NP_PIPELINE_SUBJECT_BATCH_SIZE:10000}
+#limit on select signals_legal_map
+ct.lp-pipeline.subject-batch-size=${LP_PIPELINE_SUBJECT_BATCH_SIZE:10000}
+
+#STAGES SETTINGS
+#select NAME + FID BY PHONE
+ct.np-pipeline.name-enricher-stage-concurrency=${NP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY:20}
+ct.np-pipeline.name-enricher-queue-capacity=${NP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY:20}
+
+#select FID BY INN+REG_NUM
+ct.lp-pipeline.name-enricher-stage-concurrency=${LP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY:20}
+ct.lp-pipeline.name-enricher-queue-capacity=${LP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY:20}
+
+#Save part of signal xml file and merge; only one thread
+ct.np-pipeline.merger-queue-capacity=${NP_PIPELINE_MERGER_QUEUE_CAPACITY:50}
+ct.lp-pipeline.merger-queue-capacity=${LP_PIPELINE_MERGER_QUEUE_CAPACITY:50}
+
+#STAGES SETTINGS
+#select NAME + FID BY PHONE
+ct.np-pipeline.name-updater-stage-concurrency=${NP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY:20}
+ct.np-pipeline.name-updater-queue-capacity=${NP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY:20}
+
+#select FID BY INN+REG_NUM
+ct.lp-pipeline.name-updater-stage-concurrency=${LP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY:20}
+ct.lp-pipeline.name-updater-queue-capacity=${LP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY:20}
+
+
+#select and calc flags
+ct.flag-updater.concurrency=${FLAG_UPDATER_CONCURRENCY:10}
+ct.flag-updater.queue-capacity=${FLAG_UPDATER_QUEUE_CAPACITY:100}
+
+#select and calc last rejected date
+ct.update-rejected-date-cron-expression=${LP_UPDATE_REJECTED_DATE_CRON_EXPRESSION:0 25 18 * * *}
+ct.rejected-date-updater.concurrency=${LP_REJECTED_DATE_UPDATER_CONCURRENCY:10}
+ct.rejected-date-updater.queue-capacity=${LP_REJECTED_DATE_UPDATER_QUEUE_CAPACITY:100}
+
+#Backoff settings
+ct.np-pipeline.error-attempt-count=${NP_PIPELINE_ERROR_ATTEMPT_COUNT:5}
+ct.np-pipeline.error-attempt-delay=${NP_PIPELINE_ERROR_ATTEMPT_DELAY:2S}
+ct.np-pipeline.retryable-exceptions=${NP_PIPELINE_RETRYABLE_EXCEPTIONS:org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException}
+
+ct.lp-pipeline.error-attempt-count=${LP_PIPELINE_ERROR_ATTEMPT_COUNT:5}
+ct.lp-pipeline.error-attempt-delay=${LP_PIPELINE_ERROR_ATTEMPT_DELAY:2S}
+ct.lp-pipeline.retryable-exceptions=${LP_PIPELINE_RETRYABLE_EXCEPTIONS:org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException}
+
+#Pipeline global settings
+ct.np-pipeline.global-timeout=${NP_PIPELINE_GLOBAL_TIMEOUT:2H}
+ct.np-pipeline.parallel-requests=${NP_PIPELINE_PARALLEL_REQUESTS:3}
+
+ct.lp-pipeline.global-timeout=${LP_PIPELINE_GLOBAL_TIMEOUT:2H}
+ct.lp-pipeline.parallel-requests=${LP_PIPELINE_PARALLEL_REQUESTS:3}

docker_compose/credit_tracker_config.env

@@ -1,14 +1,16 @@
 #commons
-APP_VERSION=1.20
-REGISTRY_URL=rh8-vm3.nbki.msk:5000
-JMX_PORT=
+APP_VERSION=2.0
+REGISTRY_URL=10.230.238.56:5000
+JMX_MAIN_PORT=11001
+JMX_MONITORING_PORT=11002
 JMX_HOSTNAME=
 
 #volumes
-REPORT_DIR=D:\folder_name\signals
-LOGS_DIR=D:\folder_name\signals\logs\app
-JSON_LOGS_DIR=D:\folder_name\signals\logs\json
-ACCESS_LOGS_DIR=D:\folder_name\signals\logs\access
+REPORT_MAIN_DIR=/opt/credit_tracker_docker/main/reports
+REPORT_MONITOR_DIR=/opt/credit_tracker_docker/monitor/reports
+LOGS_DIR=/opt/credit_tracker_docker/logs
+JSON_LOGS_DIR=/opt/credit_tracker_docker/logs/json
+ACCESS_LOGS_DIR=/opt/credit_tracker_docker/logs/access
 
 #APM settings
 APM_SERVICE_NAME=credit-tracker
@@ -60,6 +62,28 @@ MAX_LIFETIME_SCORING=1800000
 KEEPALIVE_TIME_SCORING=0
 LEAK_DETECTION_THRESHOLD_SCORING=0
 
+# nbch
+DB_URL_NBCH=jdbc:db2://10.230.227.102:2668/cprosd22:currentSchema=NBCH;
+USERNAME_NBCH=ros9appl
+PASSWORD_NBCH=R0s9Appl
+MAXIMUM_POOL_SIZE_NBCH=30
+MINIMUM_IDLE_NBCH=10
+CONNECTION_TIMEOUT_NBCH=10000
+VALIDATION_TIMEOUT_NBCH=5000
+IDLE_TIMEOUT_NBCH=600000
+MAX_LIFETIME_NBCH=1800000
+KEEPALIVE_TIME_NBCH=0
+LEAK_DETECTION_THRESHOLD_NBCH=0
+
+#kafka consumer
+CONSUMER_TOPIC=my-topic
+CONSUMER_CONCURENCY=8
+CONSUMER_BOOTSTRAP_SERVERS=localhost:29092
+CONSUMER_GROUP_ID=credit-tracker
+CONSUMER_AUTO_OFFSET_RESET=latest
+CONSUMER_MAX_POLL_RECORDS=125
+CONSUMER_MAX_POLL_INTERVAL_MS=300000
+
 # размер в mb
 ACCEPTED_XML_SIZE=1024
 ACCEPTED_ARCHIVE_SIZE=2048MB
@@ -71,7 +95,46 @@ SIGNALS_USER=0001ZZ000001
 # min
 MONITORING_INTERVAL=15
 # days
-UPDATE_FLAGS_INTERVAL=1
 DAYS_TO_UPDATE_FLAGS=30
 
+UPDATE_FLAGS_CRON_EXPRESSION=0 25 18 * * *
+NEW_FLAGS_CRON_EXPRESSION=0 25 18 * * *
+UPDATE_FIDS_CRON_EXPRESSION=0 10 12 * * *
+DAYS_TO_UPDATE_FIDS=30
+
+# общие настройки
+FLAG_UPDATER_CONCURRENCY=10
+FLAG_UPDATER_QUEUE_CAPACITY=100
+
+# Физ лица
+NP_SAVER_SUBJECT_BATCH_SIZE=15000
+NP_PIPELINE_SUBJECT_BATCH_SIZE=10000
+NP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=20
+NP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=20
+NP_PIPELINE_MERGER_QUEUE_CAPACITY=50
+NP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY=20
+NP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY=20
+NP_PIPELINE_ERROR_ATTEMPT_COUNT=5
+NP_PIPELINE_ERROR_ATTEMPT_DELAY=2S
+NP_PIPELINE_RETRYABLE_EXCEPTIONS=org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException}
+NP_PIPELINE_GLOBAL_TIMEOUT=2H
+NP_PIPELINE_PARALLEL_REQUESTS=3
+
+# Юр. Лица
+LP_SAVER_SUBJECT_BATCH_SIZE=15000
+LP_PIPELINE_SUBJECT_BATCH_SIZE=10000
+LP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=20
+LP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=20
+LP_PIPELINE_MERGER_QUEUE_CAPACITY=50
+LP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY=20
+LP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY=20
+LP_PIPELINE_ERROR_ATTEMPT_COUNT=5
+LP_PIPELINE_ERROR_ATTEMPT_DELAY=2S
+LP_PIPELINE_RETRYABLE_EXCEPTIONS=org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException}
+LP_PIPELINE_GLOBAL_TIMEOUT=2H
+LP_PIPELINE_PARALLEL_REQUESTS=3
+
+LP_UPDATE_REJECTED_DATE_CRON_EXPRESSION=
+LP_REJECTED_DATE_UPDATER_CONCURRENCY=10
+LP_REJECTED_DATE_UPDATER_QUEUE_CAPACITY=100
 

docker_compose/docker-compose.yml

@@ -6,18 +6,23 @@ services:
     volumes:
       - ${LOGS_DIR}:/opt/app/logs/app
       - ${JSON_LOGS_DIR}:/opt/app/logs/json
-      - ${REPORT_DIR}:/opt/app/reports
+      - shared_data:/opt/app/reports
+      - ${REPORT_MAIN_DIR}:/opt/app/reports
       - ${ACCESS_LOGS_DIR}:/opt/app/logs/access
     ports:
       - "${HOST_PORT}:${SERVER_PORT}"
     environment:
+      - APP_JAR=main
+      - XMX=4G
+      - XMS=1G
       - TZ=Europe/Moscow
-      - APM_SERVICE_NAME=${APM_SERVICE_NAME}
+      - ${JVM_EXTRA_OPTS:-} \
+      - APM_SERVICE_NAME=${APM_SERVICE_NAME}-${APP_JAR}
       - APM_SERVER_URLS=${APM_SERVER_URLS}
       - APM_ENVIRONMENT=${APM_ENVIRONMENT}
       - APM_SERVICE_NODE_NAME=${APM_SERVICE_NODE_NAME}
       - APM_SERVICE_VERSION=${APP_VERSION}
-      - JMX_PORT=${JMX_PORT}
+      - JMX_PORT=${JMX_MAIN_PORT}
       - JMX_HOSTNAME=${JMX_HOSTNAME}
       - SERVER_PORT=${SERVER_PORT}
       - DB_URL_INDIC=${DB_URL_INDIC}
@@ -53,11 +58,139 @@ services:
       - MAX_LIFETIME_SCORING=${MAX_LIFETIME_SCORING}
       - KEEPALIVE_TIME_SCORING=${KEEPALIVE_TIME_SCORING}
       - LEAK_DETECTION_THRESHOLD_SCORING=${LEAK_DETECTION_THRESHOLD_SCORING}
+      - DB_URL_NBCH=${DB_URL_NBCH}
+      - USERNAME_NBCH=${USERNAME_NBCH}
+      - PASSWORD_NBCH=${PASSWORD_NBCH}
+      - MAXIMUM_POOL_SIZE_NBCH=${MAXIMUM_POOL_SIZE_NBCH}
+      - MINIMUM_IDLE_NBCH=${MINIMUM_IDLE_NBCH}
+      - CONNECTION_TIMEOUT_NBCH=${CONNECTION_TIMEOUT_NBCH}
+      - VALIDATION_TIMEOUT_NBCH=${VALIDATION_TIMEOUT_NBCH}
+      - IDLE_TIMEOUT_NBCH=${IDLE_TIMEOUT_NBCH}
+      - MAX_LIFETIME_NBCH=${MAX_LIFETIME_NBCH}
+      - KEEPALIVE_TIME_NBCH=${KEEPALIVE_TIME_NBCH}
+      - LEAK_DETECTION_THRESHOLD_NBCH=${LEAK_DETECTION_THRESHOLD_NBCH}
+      - CONSUMER_TOPIC=${CONSUMER_TOPIC}
+      - CONSUMER_CONCURENCY=${CONSUMER_CONCURENCY}
+      - CONSUMER_BOOTSTRAP_SERVERS=${CONSUMER_BOOTSTRAP_SERVERS}
+      - CONSUMER_GROUP_ID=${CONSUMER_GROUP_ID}
+      - CONSUMER_AUTO_OFFSET_RESET=${CONSUMER_AUTO_OFFSET_RESET}
+      - CONSUMER_MAX_POLL_RECORDS=${CONSUMER_MAX_POLL_RECORDS}
+      - CONSUMER_MAX_POLL_INTERVAL_MS=${CONSUMER_MAX_POLL_INTERVAL_MS}
       - ACCEPTED_XML_SIZE=${ACCEPTED_XML_SIZE}
       - ACCEPTED_ARCHIVE_SIZE=${ACCEPTED_ARCHIVE_SIZE}
       - SIGNALS_ROOT_URI=${SIGNALS_ROOT_URI}
       - SIGNALS_TOKEN=${SIGNALS_TOKEN}
       - SIGNALS_USER=${SIGNALS_USER}
       - MONITORING_INTERVAL=${MONITORING_INTERVAL}
-      - UPDATE_FLAGS_INTERVAL=${UPDATE_FLAGS_INTERVAL}
+      - UPDATE_FLAGS_CRON_EXPRESSION=${UPDATE_FLAGS_CRON_EXPRESSION}
+      - NEW_FLAGS_CRON_EXPRESSION=${NEW_FLAGS_CRON_EXPRESSION}
+      - UPDATE_FIDS_CRON_EXPRESSION=${UPDATE_FIDS_CRON_EXPRESSION}
+      - DAYS_TO_UPDATE_FIDS=${DAYS_TO_UPDATE_FIDS}
       - DAYS_TO_UPDATE_FLAGS=${DAYS_TO_UPDATE_FLAGS}
+      - FLAG_UPDATER_CONCURRENCY=${FLAG_UPDATER_CONCURRENCY}
+      - FLAG_UPDATER_QUEUE_CAPACITY=${FLAG_UPDATER_QUEUE_CAPACITY}
+      - NP_SAVER_SUBJECT_BATCH_SIZE=${NP_SAVER_SUBJECT_BATCH_SIZE}
+      - NP_PIPELINE_SUBJECT_BATCH_SIZE=${NP_PIPELINE_SUBJECT_BATCH_SIZE}
+      - NP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=${NP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY}
+      - NP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=${NP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY}
+      - NP_PIPELINE_MERGER_QUEUE_CAPACITY=${NP_PIPELINE_MERGER_QUEUE_CAPACITY}
+      - NP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY=${NP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY}
+      - NP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY=${NP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY}
+      - NP_PIPELINE_ERROR_ATTEMPT_COUNT=${NP_PIPELINE_ERROR_ATTEMPT_COUNT}
+      - NP_PIPELINE_ERROR_ATTEMPT_DELAY=${NP_PIPELINE_ERROR_ATTEMPT_DELAY}
+      - NP_PIPELINE_RETRYABLE_EXCEPTIONS=${NP_PIPELINE_RETRYABLE_EXCEPTIONS}
+      - NP_PIPELINE_GLOBAL_TIMEOUT=${NP_PIPELINE_GLOBAL_TIMEOUT}
+      - NP_PIPELINE_PARALLEL_REQUESTS=${NP_PIPELINE_PARALLEL_REQUESTS}
+      - LP_SAVER_SUBJECT_BATCH_SIZE=${LP_SAVER_SUBJECT_BATCH_SIZE}
+      - LP_PIPELINE_SUBJECT_BATCH_SIZE=${LP_PIPELINE_SUBJECT_BATCH_SIZE}
+      - LP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=${LP_PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY}
+      - LP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=${LP_PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY}
+      - LP_PIPELINE_MERGER_QUEUE_CAPACITY=${LP_PIPELINE_MERGER_QUEUE_CAPACITY}
+      - LP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY=${LP_PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY}
+      - LP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY=${LP_PIPELINE_NAME_UPDATER_QUEUE_CAPACITY}
+      - LP_PIPELINE_ERROR_ATTEMPT_COUNT=${LP_PIPELINE_ERROR_ATTEMPT_COUNT}
+      - LP_PIPELINE_ERROR_ATTEMPT_DELAY=${LP_PIPELINE_ERROR_ATTEMPT_DELAY}
+      - LP_PIPELINE_RETRYABLE_EXCEPTIONS=${LP_PIPELINE_RETRYABLE_EXCEPTIONS}
+      - LP_PIPELINE_GLOBAL_TIMEOUT=${LP_PIPELINE_GLOBAL_TIMEOUT}
+      - LP_PIPELINE_PARALLEL_REQUESTS=${LP_PIPELINE_PARALLEL_REQUESTS}
+    command: ["/opt/app/start-credit-tracker.sh"]
+
+  credit-tracker-monitoring:
+    image: ${REGISTRY_URL}/credit-tracker:${APP_VERSION}
+    container_name: credit-tracker-monitoring
+    restart: always
+    volumes:
+      - shared_data:/opt/app/reports
+      - ${LOGS_DIR}:/opt/app/logs/app
+      - ${JSON_LOGS_DIR}:/opt/app/logs/json
+    environment:
+      - APP_JAR=monitoring
+      - TZ=Europe/Moscow
+      - XMX=4G
+      - XMS=1G
+      - APM_SERVICE_NAME=${APM_SERVICE_NAME}-${APP_JAR}
+      - APM_SERVER_URLS=${APM_SERVER_URLS}
+      - APM_ENVIRONMENT=${APM_ENVIRONMENT}
+      - APM_SERVICE_NODE_NAME=${APM_SERVICE_NODE_NAME}
+      - APM_SERVICE_VERSION=${APP_VERSION}
+      - JMX_PORT=${JMX_MONITORING_PORT}
+      - JMX_HOSTNAME=${JMX_HOSTNAME}
+      - DB_URL_INDIC=${DB_URL_INDIC}
+      - USERNAME_INDIC=${USERNAME_INDIC}
+      - PASSWORD_INDIC=${PASSWORD_INDIC}
+      - MAXIMUM_POOL_SIZE_INDIC=${MAXIMUM_POOL_SIZE_INDIC}
+      - MINIMUM_IDLE_INDIC=${MINIMUM_IDLE_INDIC}
+      - CONNECTION_TIMEOUT_INDIC=${CONNECTION_TIMEOUT_INDIC}
+      - VALIDATION_TIMEOUT_INDIC=${VALIDATION_TIMEOUT_INDIC}
+      - IDLE_TIMEOUT_INDIC=${IDLE_TIMEOUT_INDIC}
+      - MAX_LIFETIME_INDIC=${MAX_LIFETIME_INDIC}
+      - KEEPALIVE_TIME_INDIC=${KEEPALIVE_TIME_INDIC}
+      - LEAK_DETECTION_THRESHOLD_INDIC=${LEAK_DETECTION_THRESHOLD_INDIC}
+      - DB_URL_SIGNAL=${DB_URL_SIGNAL}
+      - USERNAME_SIGNAL=${USERNAME_SIGNAL}
+      - PASSWORD_SIGNAL=${PASSWORD_SIGNAL}
+      - MAXIMUM_POOL_SIZE_SIGNAL=${MAXIMUM_POOL_SIZE_SIGNAL}
+      - MINIMUM_IDLE_SIGNAL=${MINIMUM_IDLE_SIGNAL}
+      - CONNECTION_TIMEOUT_SIGNAL=${CONNECTION_TIMEOUT_SIGNAL}
+      - VALIDATION_TIMEOUT_SIGNAL=${VALIDATION_TIMEOUT_SIGNAL}
+      - IDLE_TIMEOUT_SIGNAL=${IDLE_TIMEOUT_SIGNAL}
+      - MAX_LIFETIME_SIGNAL=${MAX_LIFETIME_SIGNAL}
+      - KEEPALIVE_TIME_SIGNAL=${KEEPALIVE_TIME_SIGNAL}
+      - LEAK_DETECTION_THRESHOLD_SIGNAL=${LEAK_DETECTION_THRESHOLD_SIGNAL}
+      - DB_URL_SCORING=${DB_URL_SCORING}
+      - USERNAME_SCORING=${USERNAME_SCORING}
+      - PASSWORD_SCORING=${PASSWORD_SCORING}
+      - MAXIMUM_POOL_SIZE_SCORING=${MAXIMUM_POOL_SIZE_SCORING}
+      - MINIMUM_IDLE_SCORING=${MINIMUM_IDLE_SCORING}
+      - CONNECTION_TIMEOUT_SCORING=${CONNECTION_TIMEOUT_SCORING}
+      - VALIDATION_TIMEOUT_SCORING=${VALIDATION_TIMEOUT_SCORING}
+      - IDLE_TIMEOUT_SCORING=${IDLE_TIMEOUT_SCORING}
+      - MAX_LIFETIME_SCORING=${MAX_LIFETIME_SCORING}
+      - KEEPALIVE_TIME_SCORING=${KEEPALIVE_TIME_SCORING}
+      - LEAK_DETECTION_THRESHOLD_SCORING=${LEAK_DETECTION_THRESHOLD_SCORING}
+      - DB_URL_NBCH=${DB_URL_NBCH}
+      - USERNAME_NBCH=${USERNAME_NBCH}
+      - PASSWORD_NBCH=${PASSWORD_NBCH}
+      - MAXIMUM_POOL_SIZE_NBCH=${MAXIMUM_POOL_SIZE_NBCH}
+      - MINIMUM_IDLE_NBCH=${MINIMUM_IDLE_NBCH}
+      - CONNECTION_TIMEOUT_NBCH=${CONNECTION_TIMEOUT_NBCH}
+      - VALIDATION_TIMEOUT_NBCH=${VALIDATION_TIMEOUT_NBCH}
+      - IDLE_TIMEOUT_NBCH=${IDLE_TIMEOUT_NBCH}
+      - MAX_LIFETIME_NBCH=${MAX_LIFETIME_NBCH}
+      - KEEPALIVE_TIME_NBCH=${KEEPALIVE_TIME_NBCH}
+      - LEAK_DETECTION_THRESHOLD_NBCH=${LEAK_DETECTION_THRESHOLD_NBCH}
+      - CONSUMER_TOPIC=${CONSUMER_TOPIC}
+      - CONSUMER_CONCURENCY=${CONSUMER_CONCURENCY}
+      - CONSUMER_BOOTSTRAP_SERVERS=${CONSUMER_BOOTSTRAP_SERVERS}
+      - CONSUMER_GROUP_ID=${CONSUMER_GROUP_ID}
+      - CONSUMER_AUTO_OFFSET_RESET=${CONSUMER_AUTO_OFFSET_RESET}
+      - CONSUMER_MAX_POLL_RECORDS=${CONSUMER_MAX_POLL_RECORDS}
+      - CONSUMER_MAX_POLL_INTERVAL_MS=${CONSUMER_MAX_POLL_INTERVAL_MS}
+      - SIGNALS_ROOT_URI=${SIGNALS_ROOT_URI}
+      - SIGNALS_TOKEN=${SIGNALS_TOKEN}
+      - SIGNALS_USER=${SIGNALS_USER}
+      - MONITORING_INTERVAL=${MONITORING_INTERVAL}
+    command: ["/opt/app/start-credit-tracker.sh"]
+
+volumes:
+  shared_data:

pom.xml

@@ -10,15 +10,17 @@
     </parent>
     <groupId>ru.nbch</groupId>
     <artifactId>credit-tracker</artifactId>
-    <version>1.20</version>
+    <version>2.21.12</version>
     <name>credit_tracker</name>
     <description>Credit Tracker Application</description>
 
     <properties>
-        <java.version>17</java.version>
+        <java.version>21</java.version>
         <git.info.prefix>git_info</git.info.prefix>
         <git.info.filename>git.properties</git.info.filename>
         <registry.url>10.230.238.56:5000</registry.url>
+        <jar.main.name>main</jar.main.name>
+        <jar.monitoring.name>monitoring</jar.monitoring.name>
 <!--        <registry.url>rh8-vm3.nbki.msk:5000</registry.url>-->
     </properties>
 
@@ -36,11 +38,25 @@
             <artifactId>mybatis-spring-boot-starter</artifactId>
             <version>3.0.4</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.kafka</groupId>
+            <artifactId>spring-kafka</artifactId>
+            <version>3.3.15</version>
+        </dependency>
+        <dependency>
+            <groupId>ru.nbch</groupId>
+            <artifactId>pipeline-engine</artifactId>
+            <version>1.2</version>
+        </dependency>
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.ibm.db2</groupId>
             <artifactId>jcc</artifactId>
@@ -164,7 +180,13 @@
                                                 <files>
                                                     <file>
                                                         <source>
-                                                            ${project.build.directory}/${project.build.finalName}.jar
+                                                            ${project.build.directory}/${project.build.finalName}-${jar.main.name}.jar
+                                                        </source>
+                                                        <outputDirectory>.</outputDirectory>
+                                                    </file>
+                                                    <file>
+                                                        <source>
+                                                            ${project.build.directory}/${project.build.finalName}-${jar.monitoring.name}.jar
                                                         </source>
                                                         <outputDirectory>.</outputDirectory>
                                                     </file>
@@ -229,7 +251,27 @@
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
+
+                <executions>
+                    <execution>
+                        <id>repackage</id>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
                         <configuration>
+                            <skip>true</skip>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>build-common</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <mainClass>ru.nbch.credit_tracker.service.mode.common.CTMainApplication</mainClass>
+                            <classifier>${jar.main.name}</classifier>
+                            <attach>true</attach>
                             <excludes>
                                 <exclude>
                                     <groupId>org.projectlombok</groupId>
@@ -237,6 +279,46 @@
                                 </exclude>
                             </excludes>
                         </configuration>
+                    </execution>
+
+                    <execution>
+                        <id>build-monitoring</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <mainClass>ru.nbch.credit_tracker.service.mode.monitoring.CTMonitoringApplication</mainClass>
+                            <classifier>${jar.monitoring.name}</classifier>
+                            <attach>true</attach>
+                            <excludes>
+                                <exclude>
+                                    <groupId>org.projectlombok</groupId>
+                                    <artifactId>lombok</artifactId>
+                                </exclude>
+                            </excludes>
+                        </configuration>
+                    </execution>
+
+                    <execution>
+                        <id>build-migration</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <mainClass>ru.nbch.credit_tracker.service.mode.migration.CTMigrationApplication</mainClass>
+                            <classifier>migration</classifier>
+                            <attach>true</attach>
+                            <excludes>
+                                <exclude>
+                                    <groupId>org.projectlombok</groupId>
+                                    <artifactId>lombok</artifactId>
+                                </exclude>
+                            </excludes>
+                        </configuration>
+                    </execution>
+                </executions>
             </plugin>
             <plugin>
                 <groupId>pl.project13.maven</groupId>

src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java

@@ -1,23 +0,0 @@
-package ru.nbch.credit_tracker;
-
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.context.ConfigurableApplicationContext;
-import ru.nbch.credit_tracker.service.MonitoringService;
-import ru.nbch.credit_tracker.service.UpdateProcessingPackagesService;
-import ru.nbch.credit_tracker.service.UpdatingFlagsService;
-
-@SpringBootApplication
-public class CreditTrackerApplication {
-
-	public static void main(String[] args) {
-		ConfigurableApplicationContext ctx = SpringApplication.run(CreditTrackerApplication.class, args);
-		UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class);
-		MonitoringService monitoringService = ctx.getBean(MonitoringService.class);
-		UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class);
-		updateProcessingPackagesService.run();
-		monitoringService.run();
-		updatingFlagsService.run();
-	}
-
-}

src/main/java/ru/nbch/credit_tracker/builder/PersonFactoryBuilder.java

@@ -1,124 +0,0 @@
-package ru.nbch.credit_tracker.builder;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import ru.nbch.credit_tracker.entities.external_request.signals.Person;
-import ru.nbch.credit_tracker.service.indic.PersonService;
-import ru.nbch.credit_tracker.utils.CTUtil;
-
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
-
-@Slf4j
-public class PersonFactoryBuilder {
-    private final static DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("dd.MM.yyyy");
-
-    private final static String DEFAULT_DATE = "01.01.2000";
-    private final static String IP = "0";
-    private final static String OKSM = "999";
-    private final static String CONSENT_PURPOSE = "16";
-    private final static String REPORT_USER = "АО «НБКИ»";
-    private final static String LIABILITY = "1";
-    private final static String CONSENT_PERIOD = "3";
-    private final static String REPORT_USER_REG_NUM = "1057746710713";
-    private final static String REPORT_USER_TAX_ID = "7703548386";
-    private final static String CONSENT_HASH = "hash";
-    private final static String CONSENT_DIV_CODE = "000-000";
-    private final static String INQ_PURPOSE = "16";
-    private final static String ODT = "Иной документ";
-    private final static String CONSENT_ISSUE_AUTHORITY = "Кем выдан";
-
-
-    private PersonService.PersonData personData;
-    private Long uid;
-
-    public static PersonFactoryBuilder builder() {
-        return new PersonFactoryBuilder();
-    }
-
-    public PersonFactoryBuilder personData(PersonService.PersonData personData) {
-        this.personData = personData;
-        return this;
-    }
-
-    public PersonFactoryBuilder uid(Long uid) {
-        this.uid = uid;
-        return this;
-    }
-
-    public Person build() {
-        if (personData == null) {
-            return null;
-        }
-        PersonService.NameData nameData = personData.getName();
-        PersonService.PassportData passportData = personData.getPassport();
-        Person person = new Person();
-        person.setS(nameData.getLastName());
-        person.setN(nameData.getFirstName());
-        person.setM(StringUtils.isNotBlank(nameData.getMiddleName()) ? nameData.getMiddleName() : null);
-        person.setD(DATE_FORMAT.format(nameData.getBirthDate()));
-
-        person.setPn(getPn(passportData.getSerNum(), passportData.getIdNum()));
-        person.setPd(DATE_FORMAT.format(LocalDate.now())); // Do not use passport iss_date. Signal service could reject the package
-        person.setDt(getDt(passportData.getIdType(), passportData.getIdType2()));
-        if (person.getDt() != null && person.getDt().equals("999")) {
-            person.setOdt(ODT);
-            person.setConsentOdt(ODT);
-        }
-        person.setIp(IP);
-        person.setUid(uid);
-        person.setOksm(OKSM);
-        person.setConsentDate(DATE_FORMAT.format(LocalDate.now()));
-        person.setConsentPurpose(CONSENT_PURPOSE);
-        person.setReportUser(REPORT_USER);
-        person.setLiability(LIABILITY);
-        person.setConsentPeriod(CONSENT_PERIOD);
-        person.setAgrDate(DATE_FORMAT.format(LocalDate.now()));
-        person.setReportUserRegNum(REPORT_USER_REG_NUM);
-        person.setReportUserTaxID(REPORT_USER_TAX_ID);
-        person.setConsentHash(CONSENT_HASH);
-
-        person.setCs(person.getS());
-        person.setConsentN(person.getN());
-        person.setConsentM(person.getM());
-        person.setConsentBd(person.getD());
-        person.setConsentPn(person.getPn());
-        person.setConsentPd(person.getPd());
-        person.setConsentDivCode(CONSENT_DIV_CODE);
-        person.setConsentDt(person.getDt());
-        person.setConsentIssueAuthority(CONSENT_ISSUE_AUTHORITY);
-        person.setInqPurpose(INQ_PURPOSE);
-        return person;
-    }
-
-    private String getPn(String serNum, String idNum) {
-        String normSerNum = CTUtil.normalize(serNum);
-        String normIdNum = CTUtil.normalize(idNum);
-        return StringUtils.isNotBlank(normSerNum) ? normSerNum + normIdNum : normIdNum;
-    }
-
-    private String getDt(String idType, String idType2) {
-        if (StringUtils.isNotBlank(idType2)) {
-            return idType2;
-        } else {
-            return switch (idType) {
-                case "21" -> "21";
-                case "22" -> "22.1";
-                case "09" -> "22.2";
-                case "26" -> "23";
-                case "04" -> "24";
-                case "07" -> "25";
-                case "14", "03" -> "27";
-                case "10" -> "31";
-                case "15" -> "32";
-                case "13" -> "37";
-                case "06", "27" -> "28";
-                case "01", "02" -> "999";
-                default -> {
-                    log.warn("ID_TYPE2 is blank and no matching found for ID_TYPE {}", idType);
-                    yield "";
-                }
-            };
-        }
-    }
-}

src/main/java/ru/nbch/credit_tracker/component/MonitoringRequestValidator.java

@@ -0,0 +1,109 @@
+package ru.nbch.credit_tracker.component;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+import org.springframework.web.multipart.MultipartFile;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
+import ru.nbch.credit_tracker.enums.SignalError;
+import ru.nbch.credit_tracker.enums.XmlErrorFormat;
+import ru.nbch.credit_tracker.exceptions.SignalException;
+import ru.nbch.credit_tracker.model.MonitoringValidationError;
+import ru.nbch.credit_tracker.model.ValidationResult;
+import ru.nbch.credit_tracker.service.ValidationService;
+import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
+import ru.nbch.credit_tracker.utils.ZipUtils;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.clientExcp;
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.serverExcp;
+
+@Component
+@Slf4j
+public class MonitoringRequestValidator {
+    private final CreditTrackerSettings config;
+    private final ValidationService validationService;
+    private final StaxXmlProcessor staxXmlProcessor;
+
+    public MonitoringRequestValidator(CreditTrackerSettings config,
+                                      ValidationService validationService,
+                                      StaxXmlProcessor staxXmlProcessor) {
+        this.config = config;
+        this.validationService = validationService;
+        this.staxXmlProcessor = staxXmlProcessor;
+    }
+
+    public void validateZipContent(MultipartFile file) {
+        log.trace("Validating zip content");
+        if (file.isEmpty() || !ZipUtils.isZipFile(file)) {
+            throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
+        }
+
+        // Валидация Архива на количетсво файлов, размер и расширение
+        try (ZipInputStream zipInputStream = new ZipInputStream(file.getInputStream())) {
+            Optional<ZipEntry> xmlFileEntry = ZipUtils.extractValidEntry(zipInputStream);
+            if (xmlFileEntry.isEmpty()) {
+                throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
+            }
+            long maxXmlFileSize = config.getMaxXmlFileSize() * 1024 * 1024;
+            if (xmlFileEntry.get().getSize() > maxXmlFileSize) {
+                throw clientExcp(SignalError.ERROR_009, XmlErrorFormat.MonitoringError);
+            }
+        } catch (IOException e) {
+            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
+        }
+        log.trace("Validated zip content ok!");
+    }
+
+    public ValidationResult readAndValidateMonitoringRequest(Path filePath) {
+        log.debug("Validating monitoring request...");
+        ValidationResult validationResult = new ValidationResult();
+        AtomicBoolean subjectWasMeet = new AtomicBoolean(false);
+        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(filePath), 8 * 1024 * 1024)) {
+            staxXmlProcessor.read(in, MonitoringRequest.class,
+                request -> {
+                    try {
+                        subjectWasMeet.compareAndSet(false, true);
+                        validationService.validateMonitoringRequest(request);
+                    } catch (SignalException e) {
+                        validationResult.addError(new MonitoringValidationError(-1, "", e));
+                    } finally {
+                        if (request.getPackageName() != null && validationResult.getPackageName() == null) {
+                            validationResult.setPackageName(request.getPackageName());
+                        }
+                        if (request.getUserId() != null && validationResult.getUserId() == null) {
+                            validationResult.setUserId(request.getUserId());
+                        }
+                        if (request.getPassword() != null && validationResult.getPassword() == null) {
+                            validationResult.setPassword(request.getPassword());
+                        }
+                        if (StringUtils.isBlank(request.getSubjectType()) && validationResult.getSubjectType() == null) {
+                            validationResult.setSubjectType(1);
+                        } else if (StringUtils.isNotBlank(request.getSubjectType()) && validationResult.getSubjectType() == null) {
+                            validationResult.setSubjectType(2);
+                        }
+                        if (request.getSignalTypes() != null && !request.getSignalTypes().isEmpty()
+                            && validationResult.getSignalTypes().isEmpty()) {
+                            validationResult.setSignalTypes(request.getSignalTypes());
+                        }
+                        request.getSubjects().getSubject().clear();
+                    }
+                },
+                request -> subjectWasMeet.get() && request.getSubjects().getSubject().isEmpty(),
+                9010, "/MonitoringRequest/Subjects/Subject");
+        } catch (IOException e) {
+            throw new RuntimeException("Error while parsing and validating monitoring request", e);
+        }
+        log.debug("Validating monitoring request {}!", validationResult.getValidationErrors().isEmpty() ? "ok" : "fail");
+        return validationResult;
+    }
+}

src/main/java/ru/nbch/credit_tracker/component/RepositoryTimingAspect.java

@@ -0,0 +1,28 @@
+package ru.nbch.credit_tracker.component;
+
+import lombok.extern.slf4j.Slf4j;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.springframework.stereotype.Component;
+
+@Aspect
+@Slf4j
+@Component
+public class RepositoryTimingAspect {
+    @Around("within(@org.springframework.stereotype.Repository *)")
+    public Object timeMapper(ProceedingJoinPoint pjp) throws Throwable {
+        long start = System.nanoTime();
+        try {
+            return pjp.proceed();
+        } finally {
+            long tookMs = (System.nanoTime() - start) / 1_000_000;
+            String method = pjp.getSignature().toShortString();
+            if (tookMs > 1000) {
+                log.warn("Slow jdbc template call {} took {} ms", method, tookMs);
+            } else {
+                log.debug("Jdbc template call {} took {} ms", method, tookMs);
+            }
+        }
+    }
+}

src/main/java/ru/nbch/credit_tracker/component/SimpleMessageResolver.java

@@ -1,10 +1,12 @@
-package ru.nbch.credit_tracker.utils.error;
+package ru.nbch.credit_tracker.component;
 
+import org.springframework.stereotype.Component;
 import ru.nbch.credit_tracker.enums.EnumKeyMessage;
 
 import java.util.Arrays;
 
 
+@Component
 public class SimpleMessageResolver {
     public String resolve(EnumKeyMessage errorMessage) {
         if (errorMessage == null) return "null";

src/main/java/ru/nbch/credit_tracker/component/comparator/DeletePackageStatusComparator.java

@@ -0,0 +1,28 @@
+package ru.nbch.credit_tracker.component.comparator;
+
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.enums.Status;
+
+import java.util.Comparator;
+import java.util.function.ToIntFunction;
+
+public class DeletePackageStatusComparator {
+
+    public static Comparator<UserPackage> INSTANCE = Comparator
+            .comparingInt((ToIntFunction<UserPackage>) userPackage -> statusToInt(userPackage.getStatus()))
+            .thenComparingLong(UserPackage::getId);
+
+    public static int statusToInt(String str) {
+        try {
+            Status status = Status.valueOf(str);
+            return switch (status) {
+                case ON_MONITORING -> 1;
+                case DELETED -> 0;
+                default -> -1;
+            };
+        } catch (Throwable e) {
+            return -1;
+        }
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/component/comparator/ReportListPackageStatusComparator.java

@@ -0,0 +1,36 @@
+package ru.nbch.credit_tracker.component.comparator;
+
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.enums.Status;
+
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.function.ToIntFunction;
+
+public class ReportListPackageStatusComparator {
+
+    public static final EnumSet<Status> reportedStatuses = EnumSet.of(
+        Status.ON_MONITORING
+    );
+
+    public static Comparator<UserPackage> INSTANCE = Comparator
+            .comparingInt((ToIntFunction<UserPackage>) userPackage -> statusToInt(userPackage.getStatus()))
+            .thenComparingLong(UserPackage::getId);
+
+    /**
+     * статусы не входящие в {@code reportedStatuses}
+     * кинут ошибку в <code>ru.nbch.credit_tracker.facade.SignalFacadeV3#getReportList</code>
+     */
+    public static int statusToInt(String str) {
+        try {
+            Status status = Status.valueOf(str);
+            return switch (status) {
+                case ON_MONITORING -> 1;
+                default -> -1;
+            };
+        } catch (Throwable e) {
+            return -1;
+        }
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/config/AppConfig.java

@@ -0,0 +1,20 @@
+package ru.nbch.credit_tracker.config;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class AppConfig {
+
+    @Bean
+    public ObjectMapper entityToJsonMapper() {
+        return new ObjectMapper()
+            .registerModule(new JavaTimeModule())
+            .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
+            .setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/datasource/DatabaseConfig.java

@@ -4,6 +4,8 @@ import com.zaxxer.hikari.HikariDataSource;
 import org.springframework.boot.jdbc.DataSourceBuilder;
 import ru.nbch.credit_tracker.config.settings.DataSourceSettings;
 
+import javax.sql.DataSource;
+
 public class DatabaseConfig {
 
     protected HikariDataSource createDataSource(DataSourceSettings settings) {

src/main/java/ru/nbch/credit_tracker/config/datasource/IndicDataSourceConfig.java

@@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.io.Resource;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
 import ru.nbch.credit_tracker.config.settings.DataSourceSettings;
 
@@ -41,6 +42,7 @@ public class IndicDataSourceConfig extends DatabaseConfig {
         dataSource.setMaxLifetime(indicSetings.maxLifetime());
         dataSource.setKeepaliveTime(indicSetings.keepaliveTime());
         dataSource.setLeakDetectionThreshold(indicSetings.leakDetectionThreshold());
+        dataSource.setAutoCommit(false);
 
         try (Connection connection = dataSource.getConnection()) {
             connection.isValid(3);
@@ -60,5 +62,10 @@ public class IndicDataSourceConfig extends DatabaseConfig {
         sqlSessionFactoryBean.setTypeHandlersPackage("ru.nbch.credit_tracker.config.datasource.handlers");
         return sqlSessionFactoryBean.getObject();
     }
+
+    @Bean(name = "jdbcTemplateIndic")
+    public NamedParameterJdbcTemplate getNamedParameterJdbcTemplate(@Qualifier("indicDataSource") DataSource dataSource) {
+        return new NamedParameterJdbcTemplate(dataSource);
+    }
 }
 

src/main/java/ru/nbch/credit_tracker/config/datasource/NbchDataSourceConfig.java

@@ -0,0 +1,62 @@
+package ru.nbch.credit_tracker.config.datasource;
+
+import com.zaxxer.hikari.HikariDataSource;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.mybatis.spring.SqlSessionFactoryBean;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.Resource;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.config.settings.DataSourceSettings;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+@Slf4j
+@Configuration
+@MapperScan(
+        basePackages = "ru.nbch.credit_tracker.mapper.nbch",
+        sqlSessionFactoryRef = "sqlSessionNbch"
+)
+public class NbchDataSourceConfig extends DatabaseConfig {
+    @Value("classpath:mybatis/mapper/nbch/*.xml")
+    private Resource[] resources;
+
+    @Bean(name = "nbchDataSource")
+    public DataSource dataSource(CreditTrackerSettings config) throws SQLException {
+        DataSourceSettings signalSetings = config.getNbchDataSourceSettings();
+
+        HikariDataSource dataSource = createDataSource(signalSetings);
+        dataSource.setMaximumPoolSize(signalSetings.maximumPoolSize());
+        dataSource.setMinimumIdle(signalSetings.minimumIdle());
+        dataSource.setConnectionTimeout(signalSetings.connectionTimeout());
+        dataSource.setValidationTimeout(signalSetings.validationTimeout());
+        dataSource.setIdleTimeout(signalSetings.idleTimeout());
+        dataSource.setMaxLifetime(signalSetings.maxLifetime());
+        dataSource.setKeepaliveTime(signalSetings.keepaliveTime());
+        dataSource.setLeakDetectionThreshold(signalSetings.leakDetectionThreshold());
+
+        try (Connection connection = dataSource.getConnection()) {
+            connection.isValid(3);
+        } catch (SQLException e) {
+            log.error("Unable to connect to NBCH url={}", signalSetings.url(), e);
+            throw e;
+        }
+        log.info("Connected to NBCH: url={}", signalSetings.url());
+        return dataSource;
+    }
+
+    @Bean(name = "sqlSessionNbch")
+    public SqlSessionFactory sqlSessionFactory(@Qualifier("nbchDataSource") DataSource dataSource) throws Exception {
+        SqlSessionFactoryBean sqlSessionFactoryBean = new SqlSessionFactoryBean();
+        sqlSessionFactoryBean.setDataSource(dataSource);
+        sqlSessionFactoryBean.setMapperLocations(resources);
+        sqlSessionFactoryBean.setTypeHandlersPackage("ru.nbch.credit_tracker.config.datasource.handlers");
+        return sqlSessionFactoryBean.getObject();
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/datasource/handlers/LongTypeHandler.java

@@ -0,0 +1,41 @@
+package ru.nbch.credit_tracker.config.datasource.handlers;
+
+import java.sql.CallableStatement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.apache.ibatis.type.BaseTypeHandler;
+import org.apache.ibatis.type.JdbcType;
+import org.apache.ibatis.type.MappedTypes;
+
+@MappedTypes(Long.class)
+public class LongTypeHandler extends BaseTypeHandler<Long> {
+    @Override
+    public void setNonNullParameter(
+        PreparedStatement ps, int i, Long parameter, JdbcType jdbcType
+    ) throws SQLException {
+        ps.setString(i, parameter.toString());
+    }
+
+    @Override
+    public Long getNullableResult(ResultSet rs, String columnName) throws SQLException {
+        return parse(rs.getString(columnName));
+    }
+
+    @Override
+    public Long getNullableResult(ResultSet rs, int columnIndex) throws SQLException {
+        return parse(rs.getString(columnIndex));
+    }
+
+    @Override
+    public Long getNullableResult(CallableStatement cs, int columnIndex) throws SQLException {
+        return parse(cs.getString(columnIndex));
+    }
+
+    private Long parse(String value) {
+        if (value == null || value.isBlank()) {
+            return null;
+        }
+        return Long.valueOf(value);
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/datasource/handlers/UUIDTypeHandler.java

@@ -0,0 +1,53 @@
+package ru.nbch.credit_tracker.config.datasource.handlers;
+
+import java.sql.CallableStatement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.UUID;
+import org.apache.ibatis.type.JdbcType;
+import org.apache.ibatis.type.MappedTypes;
+import org.apache.ibatis.type.TypeHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@MappedTypes({UUID.class})
+public class UUIDTypeHandler implements TypeHandler<UUID> {
+    private static final Logger log = LoggerFactory.getLogger(UUIDTypeHandler.class);
+
+    @Override
+    public void setParameter(PreparedStatement ps, int i, UUID parameter, JdbcType jdbcType) throws SQLException {
+        if (parameter == null) {
+            ps.setObject(i, null, Types.OTHER);
+        } else {
+            ps.setObject(i, parameter.toString(), Types.OTHER);
+        }
+    }
+
+    @Override
+    public UUID getResult(ResultSet rs, String columnName) throws SQLException {
+        return toUUID(rs.getString(columnName));
+    }
+
+    @Override
+    public UUID getResult(ResultSet rs, int columnIndex) throws SQLException {
+        return toUUID(rs.getString(columnIndex));
+    }
+
+    @Override
+    public UUID getResult(CallableStatement cs, int columnIndex) throws SQLException {
+        return toUUID(cs.getString(columnIndex));
+    }
+
+    private static UUID toUUID(String val) {
+        if (val != null && !val.isEmpty() && !val.isBlank()) {
+            try {
+                return UUID.fromString(val);
+            } catch (IllegalArgumentException e) {
+                log.warn("Bad UUID found: {}", val);
+            }
+        }
+        return null;
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/kafka/KafkaConfig.java

@@ -0,0 +1,75 @@
+package ru.nbch.credit_tracker.config.kafka;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.kafka.annotation.EnableKafka;
+import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
+import org.springframework.kafka.core.ConsumerFactory;
+import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
+import org.springframework.kafka.listener.ContainerProperties;
+import org.springframework.kafka.support.serializer.JsonDeserializer;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.config.settings.KafkaConsumerSettings;
+import ru.nbch.credit_tracker.entities.external_request.kafka.PackageMessage;
+import ru.nbch.credit_tracker.model.SignalKafkaEvent;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+@EnableKafka
+@Configuration
+public class KafkaConfig {
+
+    @Bean("signalKafkaEventQueue")
+    public BlockingQueue<SignalKafkaEvent> signalKafkaEventQueue(CreditTrackerSettings settings) {
+        return new LinkedBlockingQueue<>();
+    }
+
+    @Bean
+    public ConcurrentKafkaListenerContainerFactory<String, PackageMessage> kafkaListenerContainerFactory(ConsumerFactory<String, PackageMessage> consumerFactory) {
+        ConcurrentKafkaListenerContainerFactory<String, PackageMessage> factory =
+                new ConcurrentKafkaListenerContainerFactory<>();
+        factory.setConsumerFactory(consumerFactory);
+        factory.setBatchListener(true);
+        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
+
+        return factory;
+    }
+
+    @Bean
+    //todo надо проверить конфигурацию внимательно
+    public ConsumerFactory<String, PackageMessage> consumerFactory(CreditTrackerSettings settings) {
+        KafkaConsumerSettings kafkaSettings = settings.getKafkaConsumerSettings();
+        Map<String, Object> props = new HashMap<>();
+        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaSettings.bootstrapServers());
+        props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaSettings.groupId());
+        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSettings.autoOffsetReset());
+        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, kafkaSettings.maxPollRecords());
+        props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, kafkaSettings.maxPollIntervalMs());
+
+        ObjectMapper objectMapper = new ObjectMapper();
+        objectMapper.registerModule(new JavaTimeModule());
+
+        JsonDeserializer<PackageMessage> jsonDeserializer = new JsonDeserializer<>(
+                PackageMessage.class,
+                objectMapper);
+        jsonDeserializer.addTrustedPackages("*");
+        jsonDeserializer.setUseTypeHeaders(false);
+
+
+        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+
+        return new DefaultKafkaConsumerFactory<>(
+                props,
+                new StringDeserializer(),
+                jsonDeserializer
+        );
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/config/mdc/ClearMdc.java

@@ -0,0 +1,14 @@
+package ru.nbch.credit_tracker.config.mdc;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import ru.nbch.credit_tracker.utils.LogUtil;
+
+// Annotation to only clear the MDC key at method exit
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface ClearMdc {
+    String key() default LogUtil.mdcKey;
+}

src/main/java/ru/nbch/credit_tracker/config/mdc/MdcAspect.java

@@ -1,18 +1,27 @@
 package ru.nbch.credit_tracker.config.mdc;
 
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.aspectj.lang.ProceedingJoinPoint;
 import org.aspectj.lang.annotation.Around;
 import org.aspectj.lang.annotation.Aspect;
-import org.slf4j.MDC;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
 import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
+import ru.nbch.credit_tracker.utils.LogUtil;
 
 @Slf4j
 @Aspect
 public class MdcAspect {
 
+    // For methods that only require clearing the MDC at the end
+    @Around("@annotation(clearMdc)")
+    public Object clearMdcAdvice(ProceedingJoinPoint pjp, ClearMdc clearMdc) throws Throwable {
+        try {
+            return pjp.proceed();
+        } finally {
+            LogUtil.removeMdcLogKey(clearMdc.key());
+        }
+    }
+
     // For methods that set an MDC value (from a method argument) and then clear it at the end
     @Around("@annotation(setAndClearMdc)")
     public Object setAndClearMdcAdvice(ProceedingJoinPoint pjp, SetAndClearMdc setAndClearMdc) throws Throwable {
@@ -20,30 +29,18 @@ public class MdcAspect {
         int index = setAndClearMdc.argIndex();
         Object obj = (args != null && args.length > index) ? args[index] : null;
         if (obj instanceof MonitoringRequest request) {
-            if (request.getPackageName() != null) {
-                putMdcValue(setAndClearMdc.key(), request.getPackageName());
-            }
+            LogUtil.addMdcLogKey(setAndClearMdc.key(), request.getPackageName());
         } else if (obj instanceof UserPackage userPackage) {
-            if (userPackage.getPackageName() != null) {
-                putMdcValue(setAndClearMdc.key(), userPackage.getPackageName());
-            }
+            LogUtil.addMdcLogKey(setAndClearMdc.key(), userPackage.getPackageName());
         } else if (obj instanceof String packageName) {
-            putMdcValue(setAndClearMdc.key(), packageName);
+            LogUtil.addMdcLogKey(setAndClearMdc.key(), packageName);
         } else {
             log.debug("no packageName found for for {}", pjp.toShortString());
         }
         try {
             return pjp.proceed();
         } finally {
-            MDC.remove(setAndClearMdc.key());
-        }
-    }
-
-    private void putMdcValue(String key, String value) {
-        try {
-            MDC.put(key, "[" + value + "] ");
-        } catch (Throwable e) {
-            log.error("{}", ExceptionUtils.getStackTrace(e));
+            LogUtil.removeMdcLogKey(setAndClearMdc.key());
         }
     }
 }

src/main/java/ru/nbch/credit_tracker/config/mdc/SetAndClearMdc.java

@@ -4,12 +4,13 @@ import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import ru.nbch.credit_tracker.utils.LogUtil;
 
 // Annotation to set an MDC key based on a method argument and clear it after method execution
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.METHOD)
 public @interface SetAndClearMdc {
-    String key() default "ru.nbch.credit_tracker.log.mdc.key";
+    String key() default LogUtil.mdcKey;
     // The index of the method argument to use for the MDC value (defaults to the first argument)
     int argIndex() default 0;
 }

src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java

@@ -1,5 +1,6 @@
 package ru.nbch.credit_tracker.config.rest;
 
+import java.nio.charset.Charset;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
@@ -8,24 +9,32 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.RestControllerAdvice;
 import org.springframework.web.multipart.MaxUploadSizeExceededException;
+import org.springframework.web.multipart.MultipartException;
+import ru.nbch.credit_tracker.component.SimpleMessageResolver;
 import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.user_response.errors.*;
+import ru.nbch.credit_tracker.entities.user_response.errors.CommonError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringActivePackagesError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringAllPackagesError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringDownloadError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringPackageDeleteError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringReportListError;
 import ru.nbch.credit_tracker.enums.SignalError;
 import ru.nbch.credit_tracker.exceptions.SignalClientException;
 import ru.nbch.credit_tracker.exceptions.SignalException;
 import ru.nbch.credit_tracker.exceptions.SignalServerException;
-import ru.nbch.credit_tracker.utils.error.SimpleMessageResolver;
-
-import java.nio.charset.Charset;
+import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
 
 @Slf4j
 @RestControllerAdvice
 public class GlobalExceptionHandler {
 
-    private final SimpleMessageResolver messageResolver = new SimpleMessageResolver();
+    private final SimpleMessageResolver messageResolver;
     private final String encoding;
 
-    public GlobalExceptionHandler(CreditTrackerSettings settings) {
+    public GlobalExceptionHandler(SimpleMessageResolver messageResolver,
+                                  CreditTrackerSettings settings) {
+        this.messageResolver = messageResolver;
         this.encoding = settings.getEncoding();
     }
 
@@ -42,34 +51,47 @@ public class GlobalExceptionHandler {
         return new ResponseEntity<>(error, headers, HttpStatus.BAD_REQUEST);
     }
 
-    @ExceptionHandler(value = SignalException.class, produces = MediaType.APPLICATION_XML_VALUE)
-    public ResponseEntity<CommonError> handleClientException(SignalException ex) {
+    @ExceptionHandler(value = MultipartException.class, produces = MediaType.APPLICATION_XML_VALUE)
+    public ResponseEntity<CommonError> handleSpecialException(MultipartException ex) {
+        log.warn("Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception " +
+            "[Request processing failed: {}", ExceptionUtils.getStackTrace(ex));
         HttpHeaders headers = new HttpHeaders();
         headers.setContentType(new MediaType("application", "xml", Charset.forName(encoding)));
 
-        String errorText = messageResolver.resolve(ex.getErrorMessage());
+        MonitoringError error = new MonitoringError();
+        error.setStatus("error");
+        error.setErrorCode(SignalError.ERROR_028.getCode());
+        error.setErrorText(SignalError.ERROR_028.getKey());
+        return new ResponseEntity<>(error, headers, HttpStatus.BAD_REQUEST);
+    }
 
-        CommonError commonError = getCommonError(ex);
-        commonError.setStatus("error");
-        commonError.setErrorCode(ex.getErrorMessage().getSubject().getCode());
-        commonError.setErrorText(errorText);
-
-        if (ex instanceof SignalClientException) {
-            String packageName = ((SignalClientException) ex).getPackageName();
+    @ExceptionHandler(value = SignalClientException.class, produces = MediaType.APPLICATION_XML_VALUE)
+    public ResponseEntity<CommonError> handleClientException(SignalClientException ex) {
+        CommonError err = getCommonError(ex);
+        String packageName = ex.getPackageName();
         if(packageName != null ) {
-                log.warn("Response error: {}. PackageName: {}", errorText, packageName);
+            log.warn("Response error: {}. PackageName: {}", err.getErrorText(), packageName);
         } else {
-                log.warn("Response error: {}", errorText);
-            }
-            return new ResponseEntity<>(commonError, headers, HttpStatus.BAD_REQUEST);
-        } else {
-            Exception cause = ((SignalServerException) ex).getCause();
-            log.error(cause.getMessage(), cause);
-            return new ResponseEntity<>(commonError, headers, HttpStatus.INTERNAL_SERVER_ERROR);
+            log.warn("Response error: {}", err.getErrorText());
         }
+        return new ResponseEntity<>(err, xmlHeaders(), HttpStatus.BAD_REQUEST);
     }
 
-    private static CommonError getCommonError(SignalException ex) {
+    @ExceptionHandler(value = SignalServerException.class, produces = MediaType.APPLICATION_XML_VALUE)
+    public ResponseEntity<CommonError> handleServerException(SignalServerException ex) {
+        CommonError commonError = getCommonError(ex);
+        Exception cause = ex.getCause();
+        log.error(cause.getMessage(), cause);
+        return new ResponseEntity<>(commonError, xmlHeaders(), HttpStatus.INTERNAL_SERVER_ERROR);
+    }
+
+    private HttpHeaders xmlHeaders() {
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(new MediaType("application", "xml", Charset.forName(encoding)));
+        return headers;
+    }
+
+    private CommonError getCommonError(SignalException ex) {
         CommonError commonError;
         switch (ex.getErrorFormat()) {
             case MonitoringError -> commonError = new MonitoringError();
@@ -80,6 +102,10 @@ public class GlobalExceptionHandler {
             case MonitoringPackageDeleteError -> commonError = new MonitoringPackageDeleteError();
             default -> throw new IllegalStateException("Unsupported error format: " + ex.getErrorFormat());
         }
+        commonError.setStatus("error");
+        commonError.setErrorCode(ex.getErrorMessage().getSubject().getCode());
+        String errorText = messageResolver.resolve(ex.getErrorMessage());
+        commonError.setErrorText(errorText);
         return commonError;
     }
 }

src/main/java/ru/nbch/credit_tracker/config/rest/ResponseErrorHandler.java

@@ -5,9 +5,11 @@ import org.springframework.http.client.ClientHttpResponse;
 import org.springframework.web.client.DefaultResponseErrorHandler;
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.client.HttpServerErrorException;
+import ru.nbch.credit_tracker.refactoring.Suspicios;
 
 import java.io.IOException;
 
+@Suspicios("Не используется больше")
 @Slf4j
 public class ResponseErrorHandler extends DefaultResponseErrorHandler {
     @Override

src/main/java/ru/nbch/credit_tracker/config/rest/RestConfig.java

@@ -3,8 +3,6 @@ package ru.nbch.credit_tracker.config.rest;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
-import org.springframework.web.client.RestTemplate;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
 
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -12,16 +10,9 @@ import java.util.concurrent.Executors;
 @Configuration
 public class RestConfig {
 
-    @Bean(name = "restTemplate")
-    public RestTemplate restTemplate(CreditTrackerSettings config) {
-        RestTemplate restTemplate = new RestTemplate();
-        restTemplate.setUriTemplateHandler(new RootUriTemplateHandler(config.getSignalsRootUri()));
-        restTemplate.setErrorHandler(new ResponseErrorHandler());
-        return restTemplate;
-    }
-
     @Bean(name = "executorService")
     public ExecutorService executorService() {
         return  Executors.newCachedThreadPool(new CustomizableThreadFactory("package-registration-"));
     }
+
 }

src/main/java/ru/nbch/credit_tracker/config/rest/RootUriTemplateHandler.java

@@ -1,10 +1,12 @@
 package ru.nbch.credit_tracker.config.rest;
 
 import org.springframework.web.util.DefaultUriBuilderFactory;
+import ru.nbch.credit_tracker.refactoring.Suspicios;
 
 import java.net.URI;
 import java.util.Map;
 
+@Suspicios("Не используется больше")
 public class RootUriTemplateHandler extends DefaultUriBuilderFactory {
     private final String rootUri;
 

src/main/java/ru/nbch/credit_tracker/config/settings/CreditTrackerSettings.java

@@ -20,17 +20,28 @@ public class CreditTrackerSettings {
     @NestedConfigurationProperty
     private DataSourceSettings scoringDataSourceSettings;
     @NestedConfigurationProperty
+    private DataSourceSettings nbchDataSourceSettings;
+    @NestedConfigurationProperty
     private XsdValidation xsdValidation;
-
-    private String signalsRootUri;
-    private String signalsToken;
-    private String signalsuser;
+    @NestedConfigurationProperty
+    private PipelineSettings npPipeline;
+    @NestedConfigurationProperty
+    private PipelineSettings lpPipeline;
+    @NestedConfigurationProperty
+    private FlagUpdaterSettings flagUpdater;
+    @NestedConfigurationProperty
+    private RejectedDateUpdaterSettings rejectedDateUpdater;
+    @NestedConfigurationProperty
+    private KafkaConsumerSettings kafkaConsumerSettings;
 
     private Long maxXmlFileSize;
     private String loadDir;
     private Long monitoringInterval;
-    private Long updateFlagsInterval;
-    private Long daysToUpdateFlags;
-
+    private String updateFlagsCronExpression;
+    private Integer daysToUpdateFlags;
+    private Long daysToUpdateNpFids;
+    private Long daysToUpdateLpFids;
     private String encoding;
+    private Integer npSaverSubjectBatchSize;
+    private Integer lpSaverSubjectBatchSize;
 }

src/main/java/ru/nbch/credit_tracker/config/settings/FlagUpdaterSettings.java

@@ -0,0 +1,9 @@
+package ru.nbch.credit_tracker.config.settings;
+
+import org.springframework.boot.context.properties.bind.DefaultValue;
+
+public record FlagUpdaterSettings(
+    @DefaultValue("10") Integer concurrency,
+    @DefaultValue("100") Integer queueCapacity
+) {
+}

src/main/java/ru/nbch/credit_tracker/config/settings/KafkaConsumerSettings.java

@@ -0,0 +1,11 @@
+package ru.nbch.credit_tracker.config.settings;
+
+public record KafkaConsumerSettings(
+        String topic,
+        String concurency,
+        String bootstrapServers,
+        String groupId,
+        String autoOffsetReset,
+        Integer maxPollRecords,
+        Integer maxPollIntervalMs
+) {}

src/main/java/ru/nbch/credit_tracker/config/settings/PipelineSettings.java

@@ -0,0 +1,24 @@
+package ru.nbch.credit_tracker.config.settings;
+
+import org.springframework.boot.context.properties.bind.DefaultValue;
+
+import java.time.Duration;
+import java.util.List;
+
+//X это количество потоков Save
+//тогда: Name должно быть ~3x, Fid должно быть ~2x чтобы они поспевали друг за другом?
+public record PipelineSettings(
+    @DefaultValue("5000") Integer subjectBatchSize,
+    @DefaultValue("15") Integer nameEnricherStageConcurrency,
+    @DefaultValue("20") Integer nameEnricherQueueCapacity,
+    @DefaultValue("15") Integer nameUpdaterStageConcurrency,
+    @DefaultValue("20") Integer nameUpdaterQueueCapacity,
+    @DefaultValue("50") Integer mergerQueueCapacity,
+    @DefaultValue("5") Integer errorAttemptCount,
+    @DefaultValue("2s") Duration errorAttemptDelay,
+    @DefaultValue("5m") Duration globalTimeout,
+    @DefaultValue("3") Integer parallelRequests,
+    @DefaultValue("org.springframework.dao.DataAccessException")
+    List<Class<? extends Throwable>> retryableExceptions
+) {
+}

src/main/java/ru/nbch/credit_tracker/config/settings/RejectedDateUpdaterSettings.java

@@ -0,0 +1,9 @@
+package ru.nbch.credit_tracker.config.settings;
+
+import org.springframework.boot.context.properties.bind.DefaultValue;
+
+public record RejectedDateUpdaterSettings(
+    @DefaultValue("10") Integer concurrency,
+    @DefaultValue("100") Integer queueCapacity
+) {
+}

src/main/java/ru/nbch/credit_tracker/config/task/ExecutorsConfig.java

@@ -0,0 +1,50 @@
+package ru.nbch.credit_tracker.config.task;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Configuration
+public class ExecutorsConfig {
+
+    @Bean(name = "npPipelineExecutorService",
+            destroyMethod = "shutdown")
+    public ExecutorService npPipelineExecutorService() {
+        return Executors.newFixedThreadPool(15, r -> {
+            Thread t = new Thread(r);
+            t.setName("np-pipe-starter-%d".formatted(t.threadId()));
+            return t;
+        });
+    }
+
+    @Bean(name = "lpPipelineExecutorService",
+            destroyMethod = "shutdown")
+    public ExecutorService lpPipelineExecutorService() {
+        return Executors.newFixedThreadPool(15, r -> {
+            Thread t = new Thread(r);
+            t.setName("lp-pipe-starter-%d".formatted(t.threadId()));
+            return t;
+        });
+    }
+
+    @Bean("npSaverExecutorService")
+    public ExecutorService npSaverExecutorService(CreditTrackerSettings creditTrackerSettings) {
+        return Executors.newFixedThreadPool(creditTrackerSettings.getNpPipeline().parallelRequests(), r -> {
+            Thread t = new Thread(r);
+            t.setName("np-subject-saver-%d".formatted(t.threadId()));
+            return t;
+        });
+    }
+
+    @Bean("lpSaverExecutorService")
+    public ExecutorService lpSaverExecutorService(CreditTrackerSettings creditTrackerSettings) {
+        return Executors.newFixedThreadPool(creditTrackerSettings.getLpPipeline().parallelRequests(), r -> {
+            Thread t = new Thread(r);
+            t.setName("lp-subject-saver-%d".formatted(t.threadId()));
+            return t;
+        });
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/task/PipelineStarterConfig.java

@@ -0,0 +1,77 @@
+package ru.nbch.credit_tracker.config.task;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.mapper.signal.LegalsMapMapper;
+import ru.nbch.credit_tracker.mapper.signal.PhonesMapMapper;
+import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
+import ru.nbch.credit_tracker.model.impl.EmptyTerminatePayload;
+import ru.nbch.credit_tracker.model.impl.SignalPayload;
+import ru.nbch.credit_tracker.model.impl.SubjectPayload;
+import ru.nbch.credit_tracker.model.impl.VoidPayload;
+import ru.nbch.credit_tracker.service.signal.PackageRecordsService;
+import ru.nbch.credit_tracker.service.task.SetupParam;
+import ru.nbch.credit_tracker.service.task.impl.*;
+import ru.nbch.pipeline.engine.core.PipelineBackoff;
+import ru.nbch.pipeline.engine.core.setup.IPipeLineStarter;
+import ru.nbch.pipeline.engine.core.setup.IPipeLineTerminator;
+
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+@Configuration
+public class PipelineStarterConfig {
+    @Bean
+    public Function<SetupParam, IPipeLineStarter<SubjectPayload>> pipelineStarterFactory(CreditTrackerSettings creditTrackerSettings,
+                                                                                         UserPackageMapper userPackageMapper,
+                                                                                         PackageRecordsService packageRecordsService,
+                                                                                         PhonesMapMapper phonesMapMapper,
+                                                                                         LegalsMapMapper legalsMapMapper) {
+        return (setupParam) -> {
+            if (setupParam.subjectType() == 1) {
+                return switch (setupParam.mode()) {
+                    case DEFAULT, RESEND ->
+                            new NpKickPipeLineDefaultImpl(setupParam, userPackageMapper, packageRecordsService, creditTrackerSettings);
+                    case RECALC_FID ->
+                            new NpKickRecalcPipeLine(setupParam, userPackageMapper, phonesMapMapper, packageRecordsService, creditTrackerSettings);
+                };
+            } else {
+                return switch (setupParam.mode()) {
+                    case DEFAULT, RESEND ->
+                            new LpKickPipelineDefaultImpl(setupParam, userPackageMapper, packageRecordsService, creditTrackerSettings);
+                    case RECALC_FID ->
+                            new LpKickRecalcPipeLine(setupParam, userPackageMapper, legalsMapMapper, packageRecordsService, creditTrackerSettings);
+                };
+            }
+        };
+    }
+
+    @Bean
+    public Supplier<IPipeLineTerminator<SubjectPayload, SignalPayload>> pipelineTerminatorFactory(CreditTrackerSettings creditTrackerSettings) {
+        return () -> new SetupMonitoringPipelineTerminator(creditTrackerSettings);
+    }
+
+    @Bean
+    public Supplier<IPipeLineTerminator<VoidPayload, EmptyTerminatePayload>> noOpTerminatorFactory() {
+        return NoOpTerminator::new;
+    }
+
+    @Bean
+    public PipelineBackoff npPipelineBackoff(CreditTrackerSettings creditTrackerSettings) {
+        return new PipelineBackoff(
+                creditTrackerSettings.getNpPipeline().errorAttemptCount(),
+                creditTrackerSettings.getNpPipeline().errorAttemptDelay(),
+                creditTrackerSettings.getNpPipeline().retryableExceptions()
+        );
+    }
+
+    @Bean
+    public PipelineBackoff lpPipelineBackoff(CreditTrackerSettings creditTrackerSettings) {
+        return new PipelineBackoff(
+                creditTrackerSettings.getLpPipeline().errorAttemptCount(),
+                creditTrackerSettings.getLpPipeline().errorAttemptDelay(),
+                creditTrackerSettings.getLpPipeline().retryableExceptions()
+        );
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/task/StageConfig.java

@@ -0,0 +1,54 @@
+package ru.nbch.credit_tracker.config.task;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.model.impl.SubjectPayload;
+import ru.nbch.credit_tracker.model.impl.VoidPayload;
+
+@Configuration
+public class StageConfig {
+    // NP
+
+    @Bean("npFidEnricherQueueFactory")
+    public StageQueueFactory<SubjectPayload> npFidEnricherQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getNpPipeline().nameEnricherQueueCapacity());
+    }
+
+    @Bean("npFidUpdaterQueueFactoryUpdater")
+    public StageQueueFactory<SubjectPayload> npFidUpdaterQueueFactoryUpdater(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getNpPipeline().nameUpdaterQueueCapacity());
+    }
+
+    @Bean("npNoOpQueueFactory")
+    public StageQueueFactory<VoidPayload> npNoOpQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getNpPipeline().mergerQueueCapacity());
+    }
+
+    @Bean("npFinishSetupQueueFactory")
+    public StageQueueFactory<SubjectPayload> npFinishSetupQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getNpPipeline().mergerQueueCapacity());
+    }
+
+    // LP
+
+    @Bean("lpFidEnricherQueueFactory")
+    public StageQueueFactory<SubjectPayload> lpFidEnricherQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getLpPipeline().nameEnricherQueueCapacity());
+    }
+
+    @Bean("lpFidUpdaterQueueFactory")
+    public StageQueueFactory<SubjectPayload> lpFidUpdaterQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getLpPipeline().nameUpdaterQueueCapacity());
+    }
+
+    @Bean("lpNoOpQueueFactory")
+    public StageQueueFactory<VoidPayload> lpNoOpQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getLpPipeline().mergerQueueCapacity());
+    }
+
+    @Bean("lpFinishSetupQueueFactory")
+    public StageQueueFactory<SubjectPayload> lpFinishSetupQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getLpPipeline().mergerQueueCapacity());
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/task/StageQueueFactory.java

@@ -0,0 +1,34 @@
+package ru.nbch.credit_tracker.config.task;
+
+import org.springframework.beans.factory.FactoryBean;
+import ru.nbch.pipeline.engine.core.StagePayload;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+
+public class StageQueueFactory<T extends StagePayload> implements FactoryBean<BlockingQueue<T>> {
+
+    private final int capacity;
+    public StageQueueFactory(int capacity) {
+        this.capacity = capacity;
+    }
+
+    public StageQueueFactory() {
+        this(10);
+    }
+
+    @Override
+    public BlockingQueue<T> getObject() throws Exception {
+        return new ArrayBlockingQueue<>(capacity);
+    }
+
+    @Override
+    public Class<?> getObjectType() {
+        return BlockingQueue.class;
+    }
+
+    @Override
+    public boolean isSingleton() {
+        return false;
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/xml/XmlConfig.java

@@ -1,6 +1,10 @@
 package ru.nbch.credit_tracker.config.xml;
 
 import jakarta.xml.bind.JAXBException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
@@ -12,12 +16,15 @@ import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringError;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringSetupResponse;
 import ru.nbch.credit_tracker.service.xml.jaxb.IXmlProcessor;
 import ru.nbch.credit_tracker.service.xml.jaxb.XmlProcessorImplAnyClasses;
-import ru.nbch.credit_tracker.service.xml.stax.*;
-
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Supplier;
+import ru.nbch.credit_tracker.service.xml.stax.MonitoringReportErrorsStaxtReader;
+import ru.nbch.credit_tracker.service.xml.stax.MonitoringRequestStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlReader;
+import ru.nbch.credit_tracker.service.xml.stax.OnlineDownloadReportPersonsStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.SgnlReportDeletedStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.SgnlReportStatusStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.StaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
+import ru.nbch.credit_tracker.service.xml.stax.XmlProcessorStaxImpl;
 
 @Configuration
 public class XmlConfig {
@@ -39,6 +46,7 @@ public class XmlConfig {
     public StaxXmlProcessor staxXmlProcessor() {
         List<Supplier<StaxReader<?>>> staxReader = new ArrayList<>();
         staxReader.add(SgnlReportStatusStaxReader::new);
+        staxReader.add(SgnlReportDeletedStaxReader::new);
         staxReader.add(MonitoringRequestStaxReader::new);
         staxReader.add(OnlineDownloadReportPersonsStaxReader::new);
         staxReader.add(MonitoringSignalResponseStaxXmlReader::new);

src/main/java/ru/nbch/credit_tracker/controller/SignalController.java

@@ -5,6 +5,7 @@ import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
+import ru.nbch.credit_tracker.config.mdc.ClearMdc;
 import ru.nbch.credit_tracker.entities.user_request.active_packages.MonitoringActivePackagesRequest;
 import ru.nbch.credit_tracker.entities.user_request.all_packages.MonitoringAllPackagesRequest;
 import ru.nbch.credit_tracker.entities.user_response.active_packages.MonitoringActivePackagesResponse;
@@ -12,8 +13,13 @@ import ru.nbch.credit_tracker.entities.user_response.all_packages.MonitoringAllP
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringPackageDeleteResponse;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringReportListResponse;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringSetupResponse;
+import ru.nbch.credit_tracker.enums.SignalError;
+import ru.nbch.credit_tracker.enums.XmlErrorFormat;
+import ru.nbch.credit_tracker.exceptions.SignalException;
 import ru.nbch.credit_tracker.facade.SignalFacade;
 
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.serverExcp;
+
 @RestController
 @RequestMapping("/api/v1/signal")
 public class SignalController {
@@ -25,23 +31,41 @@ public class SignalController {
     }
 
     @PostMapping(value = "/set", produces = MediaType.APPLICATION_XML_VALUE)
+    @ClearMdc
     public ResponseEntity<MonitoringSetupResponse> setupMonitoring(@RequestParam("file") MultipartFile file) {
-        return signalFacade.setupMonitoring(file);
+        signalFacade.setupMonitoringRequest(file);
+        return ResponseEntity.ok().body(new MonitoringSetupResponse());
     }
 
     @PostMapping(value = "/active_packages", consumes = MediaType.APPLICATION_XML_VALUE, produces = MediaType.APPLICATION_XML_VALUE)
     public ResponseEntity<MonitoringActivePackagesResponse> findActivePackages(@RequestBody MonitoringActivePackagesRequest activePackagesRequest) {
+        try {
             return signalFacade.findActivePackages(activePackagesRequest);
+        } catch (Exception e) {
+            throwOrWrap(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError, e);
+            return null;
+        }
     }
 
     @PostMapping(value = "/all_packages", consumes = MediaType.APPLICATION_XML_VALUE, produces = MediaType.APPLICATION_XML_VALUE)
     public ResponseEntity<MonitoringAllPackagesResponse> findAllPackages(@RequestBody MonitoringAllPackagesRequest allPackagesRequest) {
+        try {
             return signalFacade.findAllPackages(allPackagesRequest);
+        } catch (Exception e) {
+            throwOrWrap(SignalError.ERROR_013, XmlErrorFormat.MonitoringAllPackagesError, e);
+            return null;
         }
+    }
+
 
     @PostMapping(value = "/delete", params = {"username", "packageName"}, produces = MediaType.APPLICATION_XML_VALUE)
     public ResponseEntity<MonitoringPackageDeleteResponse> deletePackage(@RequestParam("username") String username, @RequestParam("packageName") String packageName) {
+        try {
             return signalFacade.deletePackage(username, packageName);
+        } catch (Exception e) {
+            throwOrWrap(SignalError.ERROR_017, XmlErrorFormat.MonitoringPackageDeleteError, e);
+            return null;
+        }
     }
 
     @GetMapping(value = "/report_list", params = {"username", "packageName"}, produces = MediaType.APPLICATION_XML_VALUE)
@@ -54,5 +78,11 @@ public class SignalController {
         return signalFacade.downloadFile(file);
     }
 
+    private static void throwOrWrap(SignalError error, XmlErrorFormat specific, Exception ex) {
+        if (ex instanceof SignalException) {
+            throw (SignalException) ex;
+        }
+        throw serverExcp(error, specific, ex);
+    }
 }
 

src/main/java/ru/nbch/credit_tracker/entities/app/AppVersion.java

@@ -0,0 +1,10 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class AppVersion {
+    private String dbVersion;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/Hit.java

@@ -0,0 +1,29 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.LocalDateTime;
+
+@Getter
+@Setter
+@Builder
+public class Hit {
+    private Long id;
+    private Long phone;
+    private Long packageId;
+    private String subjectId;
+    private Integer signalType;
+    private LocalDateTime responseSentAt;
+    private String reportFileName;
+    private Long inn;
+    private Long regNum;
+    private String busCategory;
+    private LocalDateTime inqDt;
+    private String inqP;
+    private Integer inqT;
+    private Long inqA;
+    private Integer segment;
+    private LocalDateTime recordedAt;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/LegalsMap.java

@@ -0,0 +1,22 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.*;
+
+import java.time.LocalDateTime;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class LegalsMap {
+    private Long id;
+    private Long inn;
+    private Long regNum;
+    private Integer fid;
+    private LocalDateTime searchedAt;
+    private Integer flag90days;
+    private LocalDateTime calculatedAt;
+    private LocalDateTime lastRejectedDate;
+}
+

src/main/java/ru/nbch/credit_tracker/entities/app/PackagesRecordsMap.java

@@ -0,0 +1,20 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class PackagesRecordsMap {
+    private Long id;
+    private Long packageId;
+    private String subjectId;
+    private Long phone_l;
+    private String errorText;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/PhonesMap.java

@@ -1,4 +1,4 @@
-package ru.nbch.credit_tracker.entities.dto;
+package ru.nbch.credit_tracker.entities.app;
 
 import lombok.*;
 
@@ -9,14 +9,12 @@ import java.time.LocalDateTime;
 @AllArgsConstructor
 @NoArgsConstructor
 @Builder
-public class PhoneFidMap {
+public class PhonesMap {
     private Long id;
-    private Long packageId;
-    private String subjectId;
-    private String phone;
     private Integer fid;
+    private Long phone;
     private Integer flagDebt5000;
     private Integer flag90days;
+    private LocalDateTime searchedAt;
     private LocalDateTime calculatedAt;
-    private String errorText;
 }

src/main/java/ru/nbch/credit_tracker/entities/app/SegmentMap.java

@@ -0,0 +1,15 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class SegmentMap {
+    private String memberCode;
+    private Integer segment;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/SignalLegalFidMap.java

@@ -0,0 +1,16 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.*;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class SignalLegalFidMap {
+    private Long id;
+    private Long packageId;
+    private String subjectId;
+    private Long inn;
+    private Long regNum;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/UserPackage.java

@@ -1,21 +1,25 @@
-package ru.nbch.credit_tracker.entities.dto;
+package ru.nbch.credit_tracker.entities.app;
 
-import lombok.Builder;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.*;
 
 import java.time.LocalDateTime;
 
 @Getter
 @Setter
 @Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class UserPackage {
     private Long id;
     private String userId;
     private String packageName;
+    private Integer subjectType;
+    private String signalTypes;
     private LocalDateTime createdAt;
     private String status;
     private LocalDateTime deactivatedAt;
     private String lastDownloadedReport;
     private LocalDateTime monitoringAt;
+    private String rejectStatus;
+    private String externalPackageName;
 }

src/main/java/ru/nbch/credit_tracker/entities/dto/Hit.java

@@ -1,17 +0,0 @@
-package ru.nbch.credit_tracker.entities.dto;
-
-import lombok.Getter;
-import lombok.Setter;
-
-import java.time.LocalDateTime;
-
-@Getter
-@Setter
-public class Hit {
-    private Long id;
-    private Long packageId;
-    private String subjectId;
-    private Integer signalType;
-    private LocalDateTime responseSentAt;
-    private String reportFileName;
-}

src/main/java/ru/nbch/credit_tracker/entities/external_request/kafka/InquiryDateSerializer.java

@@ -0,0 +1,36 @@
+package ru.nbch.credit_tracker.entities.external_request.kafka;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Needed only for test purposes (kafka message generator)
+ */
+public class InquiryDateSerializer extends JsonSerializer<LocalDateTime> {
+
+    @Override
+    public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
+        if (value == null) {
+            gen.writeNull();
+            return;
+        }
+
+        List<Integer> dateParts = Arrays.asList(
+                value.getYear(),
+                value.getMonthValue(),
+                value.getDayOfMonth(),
+                value.getHour(),
+                value.getMinute(),
+                value.getSecond(),
+                value.getNano()
+        );
+
+        gen.writeObject(dateParts);
+    }
+}

src/main/java/ru/nbch/credit_tracker/entities/external_request/kafka/PackageMessage.java

@@ -0,0 +1,39 @@
+package ru.nbch.credit_tracker.entities.external_request.kafka;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+@AllArgsConstructor
+@NoArgsConstructor
+@Getter
+@Setter
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PackageMessage {
+    private Integer fid;
+    private String memberCode;
+    @JsonFormat(shape = JsonFormat.Shape.ARRAY)
+    private LocalDateTime inquiryDate;
+    private String inquiryPurpose;
+    private Long inquiryAmount;
+    private String billingCode;
+    private Integer inquiryType;
+    private List<Person> persons;
+
+    @AllArgsConstructor
+    @NoArgsConstructor
+    @Getter
+    @Setter
+    public static class Person {
+        private String lastName;
+        private String firstName;
+        private String middleName;
+        private List<Integer> birthDate;
+    }
+}

src/main/java/ru/nbch/credit_tracker/entities/external_response/list_package/SgnlReportDeleted.java

@@ -0,0 +1,18 @@
+package ru.nbch.credit_tracker.entities.external_response.list_package;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Setter
+@Getter
+public class SgnlReportDeleted {
+    private Integer deleteFlag;
+
+    public Integer getDeleteFlag() {
+        return deleteFlag;
+    }
+
+    public void setDeleteFlag(Integer deleteFlag) {
+        this.deleteFlag = deleteFlag;
+    }
+}

src/main/java/ru/nbch/credit_tracker/entities/indic/Id.java

@@ -0,0 +1,24 @@
+package ru.nbch.credit_tracker.entities.indic;
+
+import java.time.LocalDate;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Id {
+    private Integer fid;
+    private String number;
+    private String serNum;
+    private String idNum;
+    private LocalDate issDate;
+    private String idType;
+    private String idType2;
+    private LocalDate fileSinceDate;
+}

src/main/java/ru/nbch/credit_tracker/entities/indic/Name.java

@@ -0,0 +1,17 @@
+package ru.nbch.credit_tracker.entities.indic;
+
+import lombok.*;
+
+import java.time.LocalDate;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Name {
+    private Integer fid;
+    private String number;
+    private LocalDate fileSinceDate;
+    private LocalDate fileSinceDatePn;
+}

src/main/java/ru/nbch/credit_tracker/entities/indic/PaymentsFlag.java

@@ -0,0 +1,11 @@
+package ru.nbch.credit_tracker.entities.indic;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class PaymentsFlag {
+    private Integer fid;
+    private Integer flag;
+}

src/main/java/ru/nbch/credit_tracker/entities/indic/RuApplication.java

@@ -0,0 +1,13 @@
+package ru.nbch.credit_tracker.entities.indic;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.LocalDate;
+
+@Getter
+@Setter
+public class RuApplication {
+    private Integer fid;
+    private LocalDate rejectedDate;
+}

src/main/java/ru/nbch/credit_tracker/entities/nbch/Source.java

@@ -0,0 +1,16 @@
+package ru.nbch.credit_tracker.entities.nbch;
+
+import lombok.*;
+
+import java.time.LocalDate;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Source {
+    private String memberCode;
+    private String category;
+    private LocalDate agrDate;
+}

src/main/java/ru/nbch/credit_tracker/entities/user_request/monitoring_request/MonitoringRequest.java

@@ -5,6 +5,9 @@ import jakarta.xml.bind.annotation.*;
 import lombok.Getter;
 import lombok.Setter;
 
+import java.util.ArrayList;
+import java.util.List;
+
 
 @Setter
 @Getter
@@ -13,6 +16,7 @@ import lombok.Setter;
         "userId",
         "password",
         "packageName",
+        "subjectType",
         "subjects"
 })
 @XmlRootElement(name = "MonitoringRequest")
@@ -24,8 +28,11 @@ public class MonitoringRequest {
     protected String password;
     @XmlElement(name = "PackageName", required = true)
     protected String packageName;
+    @XmlElement(name = "SubjectType", required = false)
+    protected String subjectType;
     @XmlElement(name = "Subjects", required = true)
     protected Subjects subjects;
+    protected List<Integer> signalTypes = null;
 
     public Subjects getSubjects() {
         if (subjects == null) {
@@ -40,6 +47,10 @@ public class MonitoringRequest {
         currentSubject = new Subject();
     }
 
+    public void createSignalTypes() {
+        signalTypes = new ArrayList<>();
+    }
+
     public Subject getCurrentSubject() {
         return currentSubject;
     }

src/main/java/ru/nbch/credit_tracker/entities/user_request/monitoring_request/Subject.java

@@ -14,7 +14,9 @@ import lombok.Setter;
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlType(name = "SubjectType", propOrder = {
     "id",
-    "clientId"
+    "clientId",
+    "inn",
+    "regNum"
 })
 public class Subject {
 
@@ -24,5 +26,10 @@ public class Subject {
     @XmlElement(name = "ClientId", required = true)
     protected String clientId;
 
+    @XmlElement(name = "Inn", required = true)
+    protected String inn;
+
+    @XmlElement(name = "RegNum", required = true)
+    protected String regNum;
 
 }

src/main/java/ru/nbch/credit_tracker/entities/user_response/errors/CommonError.java

@@ -4,4 +4,5 @@ public interface CommonError {
     void setStatus(String status);
     void setErrorCode(String errorCode);
     void setErrorText(String errorText);
+    String getErrorText();
 }

src/main/java/ru/nbch/credit_tracker/enums/EventType.java

@@ -0,0 +1,7 @@
+package ru.nbch.credit_tracker.enums;
+
+public enum EventType {
+    WRITE,
+    ROTATE,
+    STOP // используется для остановки
+}

src/main/java/ru/nbch/credit_tracker/enums/PoisonType.java

@@ -0,0 +1,7 @@
+package ru.nbch.credit_tracker.enums;
+
+public enum PoisonType {
+    NONE,
+    NORMALLY,
+    EXCEPTIONALLY
+}

src/main/java/ru/nbch/credit_tracker/enums/RejectedStatus.java

@@ -0,0 +1,17 @@
+package ru.nbch.credit_tracker.enums;
+
+public enum RejectedStatus {
+    OK("0"),
+    ERROR("1"),;
+
+
+    private final String key;
+
+    RejectedStatus(String key) {
+        this.key = key;
+    }
+
+    public String getKey() {
+        return key;
+    }
+}

src/main/java/ru/nbch/credit_tracker/enums/SignalError.java

@@ -21,7 +21,17 @@ public enum SignalError implements IEnumKeyCode {
     ERROR_019("019", "У данного пользователя не найдено ни одного пакета"),
     ERROR_020("020", "Пакет {} был отклонён при установке на мониторинг. Отчеты не формируются."),
     ERROR_021("021", "Обработка пакета {} завершилась с ошибкой, отчёты недоступны. Обратитесь в техническую поддержку."),
-    ERROR_022("022", "Пакет {} снят с мониторинга, отчёты больше недоступны.");
+    ERROR_022("022", "Пакет {} снят с мониторинга, отчёты больше недоступны."),
+    ERROR_023("023", "Пакет {} находится в обработке, удаление невозможно. Дождитесь его установки на мониторинг и повторите попытку."),
+    ERROR_024("024", "Пакет {} отклонен или при его установке произошла ошибка. Удаление невозможно."),
+    ERROR_025("025", "Пакет {} уже был удален ранее, удаление невозможно."),
+    ERROR_026("026", "Пакет {} обновляется. Удаление невозможно. Дождитесь завершения обновления и повторите попытку."),
+    ERROR_027("027", "Недопустимое значение в элементе SignalTypes"),
+    ERROR_028("028", "Некорректный формат запроса: в теле ожидается ZIP-архив"),
+    ERROR_029("029", "Элемент Inn не соответствует формату XXXXXXXXX. Id: {}"),
+    ERROR_030("030", "Элемент RegNum не соответствует формату XXXXXXXXXXXXX. Id: {}"),
+    ERROR_031("031", "Состав Subject не соответствует типу пакета"),
+    ;
 
 
     private final String code;

src/main/java/ru/nbch/credit_tracker/enums/Status.java

@@ -4,6 +4,7 @@ public enum Status {
     ON_MONITORING, // пакет стоит на мониторинге в оф. сервисе
     REJECTED, // отклонен на стороне НБКИ
     PROCESSING, // идет поиск телефона и расчет флагов по пакету
+    ACCEPTED,
     DELETED, // пакет удален
     ERROR; // произошла ошибка при постановке пакета на мониторинг в оф. сервис
 }

src/main/java/ru/nbch/credit_tracker/facade/SignalFacade.java

@@ -1,21 +1,18 @@
 package ru.nbch.credit_tracker.facade;
 
 import lombok.extern.slf4j.Slf4j;
-import org.slf4j.MDC;
-import org.springframework.aop.framework.AopContext;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.core.io.FileUrlResource;
 import org.springframework.core.io.Resource;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
-import ru.nbch.credit_tracker.config.mdc.SetAndClearMdc;
+import ru.nbch.credit_tracker.component.MonitoringRequestValidator;
+import ru.nbch.credit_tracker.component.comparator.DeletePackageStatusComparator;
+import ru.nbch.credit_tracker.component.comparator.ReportListPackageStatusComparator;
 import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
 import ru.nbch.credit_tracker.entities.user_request.active_packages.MonitoringActivePackagesRequest;
 import ru.nbch.credit_tracker.entities.user_request.all_packages.MonitoringAllPackagesRequest;
-import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
 import ru.nbch.credit_tracker.entities.user_response.active_packages.MonitoringActivePackagesResponse;
 import ru.nbch.credit_tracker.entities.user_response.active_packages.Package;
 import ru.nbch.credit_tracker.entities.user_response.active_packages.Packages;
@@ -24,253 +21,174 @@ import ru.nbch.credit_tracker.entities.user_response.all_packages.PackageAll;
 import ru.nbch.credit_tracker.entities.user_response.all_packages.PackagesAll;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringPackageDeleteResponse;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringReportListResponse;
-import ru.nbch.credit_tracker.entities.user_response.success.MonitoringSetupResponse;
 import ru.nbch.credit_tracker.enums.SignalError;
 import ru.nbch.credit_tracker.enums.Status;
 import ru.nbch.credit_tracker.enums.XmlErrorFormat;
 import ru.nbch.credit_tracker.exceptions.SignalClientException;
-import ru.nbch.credit_tracker.service.MonitoringErrorProcesingService;
-import ru.nbch.credit_tracker.service.SignalService;
+import ru.nbch.credit_tracker.model.MonitoringValidationError;
+import ru.nbch.credit_tracker.model.ValidationResult;
+import ru.nbch.credit_tracker.service.UserPackageService;
 import ru.nbch.credit_tracker.service.scoring.AuthorizationService;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
-import ru.nbch.credit_tracker.utils.CTUtil;
-import ru.nbch.credit_tracker.utils.ValidationService;
+import ru.nbch.credit_tracker.service.task.*;
+import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
+import ru.nbch.credit_tracker.service.task.impl.Saver;
+import ru.nbch.credit_tracker.utils.CtFileUtils;
+import ru.nbch.credit_tracker.utils.EnumUtil;
+import ru.nbch.credit_tracker.utils.LogUtil;
 import ru.nbch.credit_tracker.utils.ZipUtils;
+import ru.nbch.pipeline.engine.core.Pipeline;
 
 import javax.xml.stream.XMLStreamException;
-import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.time.LocalDateTime;
+import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.clientExcp;
 import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.serverExcp;
 
-@Slf4j
 @Service
-@EnableAspectJAutoProxy(exposeProxy = true)
+@Slf4j
 public class SignalFacade {
 
-    private SignalFacade self;
-
+    private final MonitoringRequestValidator monitoringRequestValidator;
     private final CreditTrackerSettings config;
+    private final UserPackageService userPackageService;
     private final AuthorizationService authorizationService;
-    private final SignalService signalService;
-    private final SignalRestService signalRestService;
-    private final MonitoringErrorProcesingService monitoringErrorProcesingService;
-    private final ValidationService validationService;
-    private final StaxXmlProcessor staxXmlProcessor;
+    private final NpPipelinePackageFabric npPipelinePackageFabric;
+    private final LpPipelinePackageFabric lpPipelinePackageFabric;
+    private final Saver saver;
+    private final NpPipelinePackageOrchestrator pipelineOrchestrator;
+    private final LpPipelinePackageOrchestrator lpPipelinePackageOrchestrator;
+    private final Pattern DOWNLOAD_FILE_PATTERN = Pattern.compile("^(?<member>[A-Za-z0-9]{6})_(?<package>[A-Za-z0-9_]+)_(?<ts>\\d{4}\\d{2}\\d{2}_\\d{2}\\d{2}\\d{2})(?:\\.xml)?\\.zip$");
 
-    private final ExecutorService executor;
-    private final long maxXmlFileSize;
-
-
-    public SignalFacade(CreditTrackerSettings config,
+    public SignalFacade(MonitoringRequestValidator monitoringRequestValidator,
+                        CreditTrackerSettings config,
+                        UserPackageService userPackageService,
                         AuthorizationService authorizationService,
-                        SignalService signalService,
-                        SignalRestService signalRestService,
-                        MonitoringErrorProcesingService monitoringErrorProcesingService,
-                        ValidationService validationService,
-                        StaxXmlProcessor staxXmlProcessor,
-                        @Qualifier("executorService") ExecutorService executor) {
+                        NpPipelinePackageFabric npPipelinePackageManager,
+                        LpPipelinePackageFabric lpPipelinePackageFabric,
+                        Saver saver, NpPipelinePackageOrchestrator pipelineOrchestrator,
+                        LpPipelinePackageOrchestrator lpPipelinePackageOrchestrator) {
+        this.monitoringRequestValidator = monitoringRequestValidator;
         this.config = config;
+        this.userPackageService = userPackageService;
         this.authorizationService = authorizationService;
-        this.signalService = signalService;
-        this.signalRestService = signalRestService;
-        this.monitoringErrorProcesingService = monitoringErrorProcesingService;
-        this.validationService = validationService;
-        this.staxXmlProcessor = staxXmlProcessor;
-        this.executor = executor;
-        this.maxXmlFileSize = config.getMaxXmlFileSize() * 1024 * 1024;
+        this.npPipelinePackageFabric = npPipelinePackageManager;
+        this.lpPipelinePackageFabric = lpPipelinePackageFabric;
+        this.saver = saver;
+        this.pipelineOrchestrator = pipelineOrchestrator;
+        this.lpPipelinePackageOrchestrator = lpPipelinePackageOrchestrator;
     }
 
-    public ResponseEntity<MonitoringSetupResponse> setupMonitoring(MultipartFile file) {
-        // первичные валидации
-        if (file.isEmpty() || !isZipFile(file)) {
-            throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
-        }
+    public void setupMonitoringRequest(MultipartFile file) {
+        log.debug("Start processing monitoring request");
+        monitoringRequestValidator.validateZipContent(file);
 
-        // Валидация Архива на количетсво файлов, размер и расширение
-        MonitoringRequest monitoringRequest;
-        Long packageId;
-        ZipInputStream zipInputStream = null;
-        String filePath;
+        log.trace("Saving monitoring request to tmp file...");
+        Path tmpReqFilePath;
         try {
-            zipInputStream = new ZipInputStream(file.getInputStream());
-            Optional<ZipEntry> xmlFileEntry = ZipUtils.extractValidEntry(zipInputStream);
-            if (xmlFileEntry.isEmpty()) {
-                throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
-            }
-            if (xmlFileEntry.get().getSize() > maxXmlFileSize) {
-                throw clientExcp(SignalError.ERROR_009, XmlErrorFormat.MonitoringError);
-            }
-
-            zipInputStream.close();
-            zipInputStream = new ZipInputStream(file.getInputStream());
-            zipInputStream.getNextEntry();
-
-            filePath = writeTmpFile(zipInputStream);
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
-        } finally {
-            if (zipInputStream != null) {
-                try {
-                    zipInputStream.close();
+            String folderPath = config.getLoadDir() + File.separator + "tmp" + File.separator;
+            tmpReqFilePath = ZipUtils.unzipAndGetFirstEntry(folderPath, file);
         } catch (IOException e) {
             throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
         }
-            }
-        }
 
-
-        // валидация xml
+        log.debug("File saved to {}", tmpReqFilePath);
+        ValidationResult validationResult = null;
         try {
-            monitoringRequest = readAndValidateMonitoringRequest(filePath);
-        } catch (Exception e) {
-            CTUtil.removeFile(filePath);
+            validationResult = monitoringRequestValidator.readAndValidateMonitoringRequest(tmpReqFilePath);
+            if (!validationResult.getValidationErrors().isEmpty()) {
+                MonitoringValidationError firstError = validationResult.getValidationErrors().getFirst();
+                userPackageService.rejectPackage(validationResult.getUserId(), validationResult.getPackageName(), validationResult.getSubjectType(), validationResult.getSignalTypes());
+                Files.delete(tmpReqFilePath);
+                throw firstError.getSignalException();
+            }
+        } catch (RuntimeException e) {
             if (e.getCause() instanceof XMLStreamException) {
                 throw clientExcp(SignalError.ERROR_011, XmlErrorFormat.MonitoringError);
             }
             if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
+                throw e;
             }
             throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
+        } catch (IOException e) {
+            log.warn("Error deleting tmp file", e);
         }
 
+        String packageName = validationResult.getPackageName();
+        Integer subjectType = validationResult.getSubjectType();
+        String userId = validationResult.getUserId();
+        String password = validationResult.getPassword();
+        List<Integer> signalTypes = validationResult.getSignalTypes();
+
+        Path parsedFilePath = tmpReqFilePath.resolveSibling(Path.of(userId.substring(0, 6) + "_" + packageName + ".xml"));
+        try {
+            Files.move(tmpReqFilePath, parsedFilePath, StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
         try {
             // проверка авторизации
-            AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(monitoringRequest.getUserId(), monitoringRequest.getPassword());
+            AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(userId, password);
             if (!authResult.isSuccess()) {
-                signalService.rejectPackage(monitoringRequest);
-                throw clientExcp(authResult.getError(), monitoringRequest.getPackageName(), XmlErrorFormat.MonitoringError);
+                userPackageService.rejectPackage(userId, packageName, subjectType, signalTypes);
+                throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringError);
             }
 
             // проверка на уникальность пакета
-            if (signalService.isPackageNotUnique(monitoringRequest)) {
-                signalService.rejectPackage(monitoringRequest);
-                throw clientExcp(SignalError.ERROR_007, monitoringRequest.getPackageName(), XmlErrorFormat.MonitoringError);
+            if (userPackageService.isPackageNotUnique(userId, packageName)) {
+                userPackageService.rejectPackage(userId, packageName, subjectType, signalTypes);
+                throw clientExcp(SignalError.ERROR_007, XmlErrorFormat.MonitoringError);
             }
 
-            packageId = signalService.registerPackage(monitoringRequest);
-            log.info("Package registered. PackageId {}. Status: {}", packageId, Status.PROCESSING.name());
+            Long packageId = userPackageService.registerPackage(userId, packageName, subjectType, signalTypes);
+            LogUtil.addMdcLogKey("%s %d".formatted(packageName, packageId));
+            log.info("Package registered. PackageId {}. Status: {}", packageId, Status.ACCEPTED.name());
+
+            if (subjectType == 1) {
+                Pipeline pipeline = npPipelinePackageFabric.build(new SetupParam(packageId, 1, PipelineMode.DEFAULT));
+                pipelineOrchestrator.startOrDelay(() -> {
+                    saver.readAndSaveNpPackage(packageId, parsedFilePath);
+                    CtFileUtils.deleteQuietly(parsedFilePath);
+                }, pipeline, packageId);
+            } else {
+                Pipeline pipeline = lpPipelinePackageFabric.build(new SetupParam(packageId, 2, PipelineMode.DEFAULT));
+                lpPipelinePackageOrchestrator.startOrDelay(() -> {
+                    saver.readAndSaveLpPackage(packageId, parsedFilePath);
+                    CtFileUtils.deleteQuietly(parsedFilePath);
+                }, pipeline, packageId);
+            }
         } catch (Exception e) {
-            CTUtil.removeFile(filePath);
+            try {
+                Files.delete(parsedFilePath);
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
             if (e instanceof SignalClientException) {
                 throw (SignalClientException) e;
             }
             throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
         }
-
-        // в отдельном потоке запускаем регистрацию субъектов, поиск фидов и расчет флагов просрочек
-        getCurrentProxy().registerFidsAndDebts(packageId, monitoringRequest.getPackageName(), filePath);
-
-        return ResponseEntity.ok().body(new MonitoringSetupResponse());
     }
 
-    private MonitoringRequest readAndValidateMonitoringRequest(String filePath) {
-        MonitoringRequest monitoringRequest;
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(filePath)), 8 * 1024 * 1024)) {
-            monitoringRequest = staxXmlProcessor.read(in, MonitoringRequest.class,
-                    request -> {
-                        try {
-                            validationService.validateMonitoringRequest(request);
-                        } catch (SignalClientException e) {
-                            if (request != null) {
-                                signalService.rejectPackage(request);
-                            }
-                            request = null;
-                            throw e;
-                        }
-                        request.getSubjects().getSubject().clear();
-                    },
-                    request -> request.getSubjects().getSubject().isEmpty()
-                    ,
-                    40000, "/MonitoringRequest/Subjects/Subject");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while parsing and validating monitoring request", e);
-        }
-        return monitoringRequest;
-    }
-
-    private String writeTmpFile(ZipInputStream zipInputStream) throws IOException {
-        String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator;
-        String filePath = folderPath + UUID.randomUUID() + "monitoring_request.xml";
-        CTUtil.createFolder(folderPath);
-
-        try (OutputStream out = Files.newOutputStream(Path.of(filePath))) {
-
-            byte[] buffer = new byte[8 * 1024 * 1024];
-            int bytesRead;
-            while ((bytesRead = zipInputStream.read(buffer)) > 0) {
-                out.write(buffer, 0, bytesRead);
-            }
-
-            return filePath;
-        }
-    }
-
-    @SetAndClearMdc(argIndex = 1)
-    public void registerFidsAndDebts(Long packageId, String packageName, String filePath) {
-        Map<String, String> context = MDC.getCopyOfContextMap();
-        executor.execute(() -> {
-            if (context != null) {
-                MDC.setContextMap(context);
-            }
-            try {
-                signalService.registerSubjects(filePath, packageId);
-                CTUtil.removeFile(filePath);
-                SignalService.UpdateResult updateResult = signalService.updateSubjects(packageId, null);
-                if (updateResult.isNoFidsFound()) {
-                    log.info("No fids found for PackageId: {} PackageName: {}. Package status ON_MONITORING. No registration in signal service.", packageId, packageName);
-                    signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-                    return;
-                }
-                if (signalRestService.setupMonitoring(packageId)) {
-                    LocalDateTime monitoringAt = LocalDateTime.now();
-                    log.info("Package was successfully registered in signal service. PackageId: {} PackageName: {}, MonitoringAt: {}", packageId, packageName, monitoringAt);
-                    signalService.updatePackageOnMonitoringSuccess(packageId, monitoringAt);
-                    monitoringErrorProcesingService.findErrors(packageId);
-                } else {
-                    log.info("Package registration was rejected by signal service. PackageId: {} PackageName: {}", packageId, packageName);
-                    signalService.updatePackageStatus(packageId, Status.ERROR);
-                }
-            } catch (Exception e) {
-                CTUtil.removeFile(filePath);
-                signalService.updatePackageStatus(packageId, Status.ERROR);
-                log.error("Ошибка при поиске фидов или расчете флагов просрочек. Package_id {}", packageId, e);
-            }
-        });
-    }
-
-    private boolean isZipFile(MultipartFile file) {
-        return file.getContentType() != null &&
-                (file.getContentType().equals("application/zip") || file.getContentType().equals("application/x-zip-compressed")) ||
-                file.getOriginalFilename() != null && file.getOriginalFilename().toLowerCase().endsWith(".zip");
-    }
-
-    public ResponseEntity<MonitoringActivePackagesResponse> findActivePackages(MonitoringActivePackagesRequest activePackagesRequest) {
-        try {
+    public ResponseEntity<MonitoringActivePackagesResponse> findActivePackages(MonitoringActivePackagesRequest activePackagesRequest) throws Exception {
         AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(activePackagesRequest.getUserId(), activePackagesRequest.getPassword());
         if (!authResult.isSuccess()) {
             throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringActivePackagesError);
         }
-            List<UserPackage> activePackages = signalService.findActivePackages(activePackagesRequest.getUserId());
 
-            if (!activePackages.isEmpty()) {
+        List<UserPackage> activePackages = userPackageService.findActivePackages(activePackagesRequest.getUserId());
+        if (activePackages.isEmpty()) {
+            throw clientExcp(SignalError.ERROR_015, XmlErrorFormat.MonitoringActivePackagesError);
+        }
+
         MonitoringActivePackagesResponse response = new MonitoringActivePackagesResponse();
         response.setStatus("OK");
         response.setMembercode(activePackagesRequest.getUserId().substring(0, 6));
@@ -283,26 +201,19 @@ public class SignalFacade {
         });
         response.setPackages(packages);
         return ResponseEntity.ok(response);
-            } else {
-                throw clientExcp(SignalError.ERROR_015, XmlErrorFormat.MonitoringActivePackagesError);
-            }
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError, e);
-        }
     }
 
-    public ResponseEntity<MonitoringAllPackagesResponse> findAllPackages(MonitoringAllPackagesRequest allPackagesRequest) {
-        try {
+    public ResponseEntity<MonitoringAllPackagesResponse> findAllPackages(MonitoringAllPackagesRequest allPackagesRequest) throws Exception {
         AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(allPackagesRequest.getUserId(), allPackagesRequest.getPassword());
         if (!authResult.isSuccess()) {
             throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringAllPackagesError);
         }
-            List<UserPackage> activePackages = signalService.findAllPackages(allPackagesRequest.getUserId());
 
-            if (!activePackages.isEmpty()) {
+        List<UserPackage> activePackages = userPackageService.findAllPackages(allPackagesRequest.getUserId());
+        if (activePackages.isEmpty()) {
+            throw clientExcp(SignalError.ERROR_019, XmlErrorFormat.MonitoringAllPackagesError);
+        }
+
         MonitoringAllPackagesResponse response = new MonitoringAllPackagesResponse();
         response.setStatus("OK");
         response.setMembercode(allPackagesRequest.getUserId().substring(0, 6));
@@ -311,65 +222,80 @@ public class SignalFacade {
             PackageAll pack = new PackageAll();
             pack.setPackageName(data.getPackageName());
             pack.setCreatedAt(data.getCreatedAt() != null ? data.getCreatedAt().toString() : null);
-                    pack.setStatus(data.getStatus());
-                    if (data.getStatus().equals(Status.DELETED.name())) {
+            Status packageStatus = EnumUtil.getEnum(Status.class, data.getStatus());
+            Status statusForClient = packageStatus == Status.ACCEPTED ? Status.PROCESSING : packageStatus;
+            assert statusForClient != null;
+            pack.setStatus(statusForClient.name());
+            if (packageStatus == Status.DELETED) {
                 pack.setDeactivatedAt(data.getDeactivatedAt() != null ? data.getDeactivatedAt().toString() : null);
             }
             packages.getPackages().add(pack);
         });
         response.setPackages(packages);
         return ResponseEntity.ok(response);
-            } else {
-                throw clientExcp(SignalError.ERROR_019, XmlErrorFormat.MonitoringAllPackagesError);
-            }
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringAllPackagesError, e);
-        }
     }
 
     public ResponseEntity<MonitoringPackageDeleteResponse> deletePackage(String username, String packageName) {
-        try {
-            if (!signalService.isPackageExists(username, packageName)) {
-                throw clientExcp(SignalError.ERROR_016, XmlErrorFormat.MonitoringPackageDeleteError, packageName);
+        UserPackage packageToDelete = userPackageService.findByMemberCodeAndPackageName(username, packageName)
+            .stream()
+            .max(DeletePackageStatusComparator.INSTANCE)
+            .orElseThrow(()
+                -> clientExcp(SignalError.ERROR_016, XmlErrorFormat.MonitoringPackageDeleteError, packageName));
+
+        Status status = EnumUtil.getEnum(Status.class, packageToDelete.getStatus());
+        if (Status.ON_MONITORING != status) {
+            throw clientExcp(deleteStatusError(status), XmlErrorFormat.MonitoringPackageDeleteError, packageName);
         }
 
         LocalDateTime deactivatedAt = LocalDateTime.now();
-            signalService.deletePackage(username, packageName, deactivatedAt);
-
+        userPackageService.deletePackageById(packageToDelete.getId(), deactivatedAt);
         MonitoringPackageDeleteResponse response = new MonitoringPackageDeleteResponse();
         response.setStatus("Deleted");
         response.setPackageName(packageName);
         response.setDeactivatedAt(deactivatedAt.toString());
 
-            signalRestService.deletePackage(username, packageName);
         log.info("Package {} deleted", packageName);
         return ResponseEntity.ok(response);
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_017, XmlErrorFormat.MonitoringPackageDeleteError, e);
     }
+
+
+    private SignalError deleteStatusError(Status status) {
+        if (status == null) return SignalError.ERROR_017;
+        return switch (status) {
+            case PROCESSING, ACCEPTED -> SignalError.ERROR_023;
+            case REJECTED, ERROR -> SignalError.ERROR_024;
+            case DELETED -> SignalError.ERROR_025;
+            default -> throw new IllegalStateException("Unexpected value: " + status);
+        };
     }
 
     public ResponseEntity<MonitoringReportListResponse> getReportList(String username, String packageName) {
         try {
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.PROCESSING)) {
+            UserPackage userPackage = userPackageService
+                .findByMemberCodeAndPackageName(username, packageName)
+                .stream()
+                .max(ReportListPackageStatusComparator.INSTANCE)
+                .orElse(null);
+            if (userPackage == null) {
+                throw clientExcp(SignalError.ERROR_012, XmlErrorFormat.MonitoringReportListError, packageName);
+            }
+            Status status = EnumUtil.getEnum(Status.class, userPackage.getStatus());
+            if (status == null) {
+                throw clientExcp(SignalError.ERROR_017, XmlErrorFormat.MonitoringReportListError);
+            }
+            if (Arrays.asList(Status.PROCESSING, Status.ACCEPTED).contains(status)) {
                 throw clientExcp(SignalError.ERROR_018, XmlErrorFormat.MonitoringReportListError, packageName);
             }
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.REJECTED)) {
+            if (status.equals(Status.REJECTED)) {
                 throw clientExcp(SignalError.ERROR_020, XmlErrorFormat.MonitoringReportListError, packageName);
             }
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.ERROR)) {
+            if (status.equals(Status.ERROR)) {
                 throw clientExcp(SignalError.ERROR_021, XmlErrorFormat.MonitoringReportListError, packageName);
             }
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.DELETED)) {
+            if (status.equals(Status.DELETED)) {
                 throw clientExcp(SignalError.ERROR_022, XmlErrorFormat.MonitoringReportListError, packageName);
             }
-            if (!signalService.isPackagesByStatusExist(username, packageName, Status.ON_MONITORING)) {
+            if (!ReportListPackageStatusComparator.reportedStatuses.contains(status)) {
                 throw clientExcp(SignalError.ERROR_012, XmlErrorFormat.MonitoringReportListError, packageName);
             }
 
@@ -377,6 +303,7 @@ public class SignalFacade {
             String filePrefix = (memberCode + "_" + packageName).toUpperCase();
             String dirPath = config.getLoadDir() + File.separator + memberCode;
             File folder = new File(dirPath);
+            log.debug("searching for reports from: {}", folder.getAbsolutePath());
 
             MonitoringReportListResponse.Reports reports = new MonitoringReportListResponse.Reports();
             if (folder.exists() && folder.isDirectory()) {
@@ -403,15 +330,21 @@ public class SignalFacade {
 
     public ResponseEntity<Resource> downloadFile(String fileName) {
         try {
-            String memberCode = CTUtil.getMemberCodeFromFileName(fileName);
+            Matcher matcher = DOWNLOAD_FILE_PATTERN.matcher(fileName);
+            String memberCode = null;
+            if (matcher.matches()) {
+                memberCode = matcher.group("member");
+            }
+
             if (memberCode == null) {
                 throw clientExcp(SignalError.ERROR_014, XmlErrorFormat.MonitoringDownloadError, fileName);
             }
-            String dirPath = config.getLoadDir() + File.separator + memberCode;
-            File folder = new File(dirPath);
+            Path reportRootPath = Path.of(config.getLoadDir(), memberCode);
+            File folder = reportRootPath.toFile();
 
             if (folder.exists() && folder.isDirectory()) {
-                File file = new File(dirPath + File.separator + fileName);
+                Path filePath = reportRootPath.resolve(fileName);
+                File file = filePath.toFile();
                 if (file.exists() && file.isFile()) {
                     return ResponseEntity.ok().body(new FileUrlResource(file.getPath()));
                 }
@@ -426,11 +359,4 @@ public class SignalFacade {
             throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringDownloadError, e);
         }
     }
-
-    private SignalFacade getCurrentProxy() {
-        if (self == null) {
-            self = (SignalFacade) AopContext.currentProxy();
-        }
-        return self;
-    }
 }

src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java

@@ -1,19 +0,0 @@
-package ru.nbch.credit_tracker.mapper.indic;
-
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.service.indic.FlagsService;
-
-import java.time.LocalDate;
-import java.util.List;
-
-@Mapper
-public interface FlagsMapper {
-
-    List<FlagsService.FlagFidData>  findFlagsDebt5000(@Param("fids") List<Integer> fids);
-
-    List<FlagsService.FlagFidData>  findFlag90ByPastDueArrear(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
-
-    List<FlagsService.FlagFidData>  findFlag90ByRuPaymt(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
-
-}

src/main/java/ru/nbch/credit_tracker/mapper/indic/IndicFlagsMapper.java

@@ -0,0 +1,22 @@
+package ru.nbch.credit_tracker.mapper.indic;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.indic.PaymentsFlag;
+import ru.nbch.credit_tracker.entities.indic.RuApplication;
+
+import java.time.LocalDate;
+import java.util.List;
+
+@Mapper
+public interface IndicFlagsMapper {
+    List<PaymentsFlag> findFlagsDebt5000(@Param("fids") List<Integer> fids);
+
+    List<PaymentsFlag> findFlag90ByPastDueArrear(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
+
+    List<PaymentsFlag> findFlag90ByRuPaymt(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
+
+    List<PaymentsFlag> findFlag90ByRuAllArrears(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
+
+    List<RuApplication> findLastRejecteDates(@Param("fids") List<Integer> fids);
+}

src/main/java/ru/nbch/credit_tracker/mapper/indic/PersonMapper.java

@@ -2,14 +2,17 @@ package ru.nbch.credit_tracker.mapper.indic;
 
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.service.indic.PersonService;
+import ru.nbch.credit_tracker.entities.indic.Name;
+import ru.nbch.credit_tracker.model.FidIdData;
 
 import java.util.List;
 
 @Mapper
 public interface PersonMapper {
 
-    List<PersonService.NameData> findNameData(@Param("fids") List<Integer> fids);
+    List<Name> findNameData(@Param("phones") List<String> phones);
 
-    List<PersonService.PassportData> findPassportData(@Param("fids") List<Integer> fids);
+    List<FidIdData> findFidsByInn(@Param("inns") List<String> inns);
+
+    List<FidIdData> findFidsByRegNum(@Param("regNums") List<String> regNums);
 }

src/main/java/ru/nbch/credit_tracker/mapper/indic/PhoneMapper.java

@@ -1,12 +0,0 @@
-package ru.nbch.credit_tracker.mapper.indic;
-
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.service.indic.PhoneService;
-
-import java.util.List;
-
-@Mapper
-public interface PhoneMapper {
-    List<PhoneService.FidsData> findFids(@Param("phones") List<String> phones);
-}

src/main/java/ru/nbch/credit_tracker/mapper/nbch/SourceMapper.java

@@ -0,0 +1,16 @@
+package ru.nbch.credit_tracker.mapper.nbch;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.nbch.Source;
+
+import java.util.List;
+
+@Mapper
+public interface SourceMapper {
+
+    List<Source> findBusCategories(@Param("memberCodes") List<String> memberCodes);
+
+    String findBusCategory(@Param("memberCode") String memberCode);
+
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/AppVersionMapper.java

@@ -0,0 +1,9 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import org.apache.ibatis.annotations.Mapper;
+import ru.nbch.credit_tracker.entities.app.AppVersion;
+
+@Mapper
+public interface AppVersionMapper {
+    AppVersion getAppVersion();
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/HitsMapper.java

@@ -2,7 +2,7 @@ package ru.nbch.credit_tracker.mapper.signal;
 
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.entities.dto.Hit;
+import ru.nbch.credit_tracker.entities.app.Hit;
 
 import java.util.List;
 

src/main/java/ru/nbch/credit_tracker/mapper/signal/LegalsFidMapMapper.java

@@ -0,0 +1,19 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.SignalLegalFidMap;
+
+import java.util.List;
+
+@Mapper
+public interface LegalsFidMapMapper {
+
+    List<Long> registerJson(@Param("json") String json);
+
+    List<SignalLegalFidMap> findFullLegalFidMap(@Param("packageId") Long packageId,
+                                                @Param("limit") int limit,
+                                                @Param("offset") int offset);
+
+    Long deleteByPackageId(@Param("packageId") Long id);
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/LegalsMapMapper.java

@@ -0,0 +1,43 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.LegalsMap;
+
+import java.util.List;
+
+@Mapper
+public interface LegalsMapMapper {
+
+    void registerJson(@Param("json") String json);
+
+    List<LegalsMap> findByFids(@Param("fids") List<Integer> fids);
+
+    void updateFlagCachedData(@Param("json") String json);
+
+    void updateLastRejectedDateCache(@Param("json")  String json);
+
+    List<LegalsMap> findEligibleForFlagsUpdate(@Param("lastId") Long lastId,
+                                               @Param("interval") Integer interval,
+                                               @Param("batchSize") Integer batchSize);
+
+    List<LegalsMap> findEligibleForRejectedDatesToUpdate(@Param("lastId") Long lastId,
+                                                         @Param("batchSize") Integer batchSize);
+
+    List<LegalsMap> findEligibleForFlagsInsert(@Param("lastId") Long lastId,
+                                               @Param("batchSize") Integer batchSize);
+
+    List<LegalsMap> searchedAtLessThan(@Param("daysCount") Integer daysCount,
+                                       @Param("batchSize") int batchSize,
+                                       @Param("lastId") Long lastId);
+
+    List<LegalsMap> findCachedByInnAndRegNum(@Param("json") String json);
+
+    void updateSearchedAt(@Param("json") String json);
+
+    void resetFidAndFlagsByInnAndRegNum(@Param("json") String json);
+
+    void updateFidCachedData(@Param("json") String json);
+
+    void deleteUnusedMaps();
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/PackagesRecordsMapper.java

@@ -0,0 +1,31 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.PackagesRecordsMap;
+import ru.nbch.credit_tracker.model.UserPackageLegalData;
+import ru.nbch.credit_tracker.model.UserPackagePhoneData;
+
+import java.util.List;
+
+@Mapper
+public interface PackagesRecordsMapper {
+
+    List<Long> registerJson(@Param("json") String json);
+
+    List<PackagesRecordsMap> findFullPhoneFidMap(@Param("packageId") Long packageId,
+                                                 @Param("limit") int limit,
+                                                 @Param("offset") int offset);
+
+    List<UserPackagePhoneData> selectUserPackagePhoneData(@Param("json") String json);
+
+    List<UserPackageLegalData> selectUserPackageLegalData(@Param("json") String json);
+
+    Long deleteByPackageId(@Param("packageId") Long id);
+
+    PackagesRecordsMap findById(Long id);
+
+    List<PackagesRecordsMap> findByPhone(@Param("phone") Long phone);
+
+    List<PackagesRecordsMap> findByPhones(@Param("phones") List<Long> phones);
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java

@@ -1,24 +0,0 @@
-package ru.nbch.credit_tracker.mapper.signal;
-
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.user_request.monitoring_request.Subject;
-
-import java.util.List;
-
-@Mapper
-public interface PhoneFidMapMapper {
-
-    void registerSubjects(@Param("subjects") List<Subject> subjects, @Param("packageId") Long packageId);
-
-    List<PhoneFidMap> findFullPhoneFidMap(@Param("packageId") Long packageId, @Param("offset") int offset, @Param("limit") int limit);
-
-    void updatePhoneFidMaps(@Param("subjects") List<PhoneFidMap> subjects);
-
-    void updatePhoneFidMapsErrors(@Param("subjects") List<PhoneFidMap> subjects);
-
-    PhoneFidMap findPhoneFidMapById(@Param("id") Long packageId);
-
-    boolean hasNonNullFids(@Param("packageId") Long packageId);
-}

src/main/java/ru/nbch/credit_tracker/mapper/signal/PhonesMapMapper.java

@@ -0,0 +1,42 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.PhonesMap;
+
+import java.util.List;
+
+@Mapper
+public interface PhonesMapMapper {
+
+    List<PhonesMap> getCachedData(@Param("phones") List<Long> phones);
+
+    List<PhonesMap> searchedAtLessThan(@Param("daysCount") Integer daysCount,
+                                       @Param("batchSize") int batchSize,
+                                       @Param("lastId") Long lastId);
+
+    PhonesMap findByPhone(@Param("phone") Long phone);
+
+    List<PhonesMap> findEligibleForFlagsUpdate(@Param("lastId") Long lastId,
+                                               @Param("interval") Integer interval,
+                                               @Param("batchSize") Integer batchSize);
+
+    List<PhonesMap> findEligibleForFlagsInsert(@Param("lastId") Long lastId,
+                                               @Param("batchSize") Integer batchSize);
+
+    void updateNameCachedData(@Param("json") String json);
+
+    void updateFlagCachedData(@Param("json") String json);
+
+    void resetFidAndPD(@Param("json") String json);
+
+    void updateSearchedAt(@Param("json") String json);
+
+    void registerJson(@Param("json") String json);
+
+    void deleteUnusedPhones();
+
+    List<PhonesMap> findByFid(@Param("fid") Integer fid);
+
+    List<PhonesMap> findByFids(@Param("fids") List<Integer> fids);
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/SegmentMapMapper.java

@@ -0,0 +1,13 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import org.apache.ibatis.annotations.Mapper;
+import ru.nbch.credit_tracker.entities.app.SegmentMap;
+
+import java.util.List;
+
+@Mapper
+public interface SegmentMapMapper {
+
+    List<SegmentMap> findAllSegmentMaps();
+
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/UserPackageMapper.java

@@ -2,7 +2,7 @@ package ru.nbch.credit_tracker.mapper.signal;
 
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
 
 import java.time.LocalDateTime;
 import java.util.List;
@@ -10,27 +10,36 @@ import java.util.List;
 @Mapper
 public interface UserPackageMapper {
 
-    boolean isPackageNotUnique(@Param("userId") String userId, @Param("packageName") String packageName);
+    boolean isPackageNotUnique(@Param("userId") String userId,
+                               @Param("packageName") String packageName);
 
-    boolean isPackageExists(@Param("userId") String userId, @Param("packageName") String packageName, @Param("status") String status);
-
-    boolean hasActivePackages(@Param("userId") String userId, @Param("packageName")  String packageName, @Param("statuses") List<String> statuses);
-
-    void deletePackage(@Param("userId") String userId, @Param("packageName") String packageName, @Param("status") String newStatus, @Param("previousStatus") String previousStatus, @Param("deactivatedAt") LocalDateTime deactivatedAt);
+    void deletePackageById(@Param("id") Long id,
+                           @Param("status") String newStatus,
+                           @Param("deactivatedAt") LocalDateTime deactivatedAt);
 
     Long insertPackage(@Param("userPackage") UserPackage userPackage);
 
-    void updatePackageStatus(@Param("packageId") Long packageId, @Param("status") String status);
+    void updatePackageStatus(@Param("packageId") Long packageId,
+                             @Param("status") String status);
 
-    void updatePackageLastDownloadedReport(@Param("id") Long id, @Param("fileName") String fileName);
-
-    void updatePackageOnMonitoringSuccess(@Param("id") Long id, @Param("monitoringAt") LocalDateTime monitoringAt);
+    void updatePackageOnMonitoringSuccess(@Param("id") Long id,
+                                          @Param("monitoringAt") LocalDateTime monitoringAt);
 
     UserPackage findPackage(@Param("id") Long id);
 
-    List<UserPackage> findUserPackages(@Param("userId") String userId, @Param("statuses") List<String> status);
+    List<UserPackage> findUserPackages(@Param("userId") String userId,
+                                       @Param("statuses") List<String> status);
 
-    List<UserPackage> findAllPackagesByStatus(@Param("status")  String status);
+    List<UserPackage> findAllPackagesByStatus(@Param("statuses") String... statuses);
 
     List<UserPackage> findAllPackagesByUserId(@Param("userId") String userId);
+
+    UserPackage findByPackageName(@Param("packageName") String packageName);
+
+    List<UserPackage> findOnMonitoringPackages();
+
+    List<UserPackage> findByMemberCodeAndPackageName(@Param("memberCode") String memberCode,
+                                                     @Param("packageName") String packageName);
+
+    void updatePackageLastDownloadedReport(@Param("json") String json);
 }

src/main/java/ru/nbch/credit_tracker/model/FidIdData.java

@@ -0,0 +1,11 @@
+package ru.nbch.credit_tracker.model;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class FidIdData {
+    private Integer fid;
+    private String idNum;
+}

src/main/java/ru/nbch/credit_tracker/model/FlagData.java

@@ -0,0 +1,20 @@
+package ru.nbch.credit_tracker.model;
+
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.LocalDate;
+
+@Getter
+@Setter
+@Builder
+public class FlagData {
+    private Long phone;
+    private Long inn;
+    private Long regNum;
+    private Integer flag_debt5000;
+    private Integer flag_90_12;
+    private LocalDate lastRejectedDate;
+}

src/main/java/ru/nbch/credit_tracker/model/IUserPackageData.java

@@ -0,0 +1,13 @@
+package ru.nbch.credit_tracker.model;
+
+public interface IUserPackageData {
+    Long getPackageId();
+
+    String getUserId();
+
+    String getSignalTypes();
+
+    String getSubjectId();
+
+    String getPackageName();
+}

src/main/java/ru/nbch/credit_tracker/model/MonitoringValidationError.java

@@ -0,0 +1,30 @@
+package ru.nbch.credit_tracker.model;
+
+import ru.nbch.credit_tracker.exceptions.SignalException;
+
+public class MonitoringValidationError {
+    private final int line;
+    private final String message;
+    private final SignalException signalException;
+
+
+    public MonitoringValidationError(int line,
+                                     String message,
+                                     SignalException signalException) {
+        this.line = line;
+        this.message = message;
+        this.signalException = signalException;
+    }
+
+    public int getLine() {
+        return line;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public SignalException getSignalException() {
+        return signalException;
+    }
+}

src/main/java/ru/nbch/credit_tracker/model/SignalKafkaEvent.java

@@ -0,0 +1,32 @@
+package ru.nbch.credit_tracker.model;
+
+import lombok.*;
+import ru.nbch.credit_tracker.entities.external_request.kafka.PackageMessage;
+import ru.nbch.credit_tracker.enums.EventType;
+
+import java.time.LocalDateTime;
+import java.util.Map;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SignalKafkaEvent {
+    private EventType eventType;
+    private Map<String, String> reportNames;
+
+    private UserPackagePhoneData userPackagePhoneData;
+    private UserPackageLegalData userPackageLegalData;
+    private PackageMessage packageMessage;
+
+    private String subjectId;
+    private Integer flagDebt5000;
+    private Integer flag90days;
+    private String signalType;
+    private String busCategory;
+    private Integer segment;
+    private LocalDateTime lastRejectedDate;
+
+    private boolean natural;
+}

src/main/java/ru/nbch/credit_tracker/model/SubjectProfile.java

@@ -0,0 +1,23 @@
+package ru.nbch.credit_tracker.model;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+public class SubjectProfile {
+    // common field
+    private String identifier;
+    private Long mapId;
+    private Integer fid;
+    private FlagData flagData;
+
+    // np fields
+    private Long phone;
+
+    // lp fields
+    private Long inn;
+    private Long regNum;
+}

src/main/java/ru/nbch/credit_tracker/model/UserPackageLegalData.java

@@ -0,0 +1,19 @@
+package ru.nbch.credit_tracker.model;
+
+import lombok.*;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class UserPackageLegalData implements IUserPackageData {
+    private Long packageId;
+    private String signalTypes;
+    private String subjectId;
+    private String userId;
+    private String packageName;
+
+    private Long inn;
+    private Long regNum;
+}

src/main/java/ru/nbch/credit_tracker/model/UserPackagePhoneData.java

@@ -0,0 +1,17 @@
+package ru.nbch.credit_tracker.model;
+
+import lombok.*;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class UserPackagePhoneData implements IUserPackageData {
+    private Long packageId;
+    private String signalTypes;
+    private Long phone;
+    private String subjectId;
+    private String userId;
+    private String packageName;
+}

src/main/java/ru/nbch/credit_tracker/model/ValidationResult.java

@@ -0,0 +1,69 @@
+package ru.nbch.credit_tracker.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ValidationResult {
+    private String packageName;
+    private String userId;
+    private String password;
+    private Integer subjectType;
+    private List<Integer> signalTypes = new ArrayList<>();
+    private List<MonitoringValidationError> validationErrors = new ArrayList<>();
+
+    public void setPackageName(String packageName) {
+        this.packageName = packageName;
+    }
+
+    public void setUserId(String userId) {
+        this.userId = userId;
+    }
+
+    public void setValidationErrors(List<MonitoringValidationError> validationErrors) {
+        this.validationErrors = validationErrors;
+    }
+
+    public ValidationResult() {
+    }
+
+    public String getPackageName() {
+        return packageName;
+    }
+
+    public String getUserId() {
+        return userId;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public Integer getSubjectType() {
+        return subjectType;
+    }
+
+    public void setSubjectType(Integer subjectType) {
+        this.subjectType = subjectType;
+    }
+
+    public List<Integer> getSignalTypes() {
+        return signalTypes;
+    }
+
+    public void setSignalTypes(List<Integer> signalTypes) {
+        this.signalTypes = signalTypes;
+    }
+
+    public List<MonitoringValidationError> getValidationErrors() {
+        return validationErrors;
+    }
+
+    public void  addError(MonitoringValidationError error) {
+        this.validationErrors.add(error);
+    }
+}
+

src/main/java/ru/nbch/credit_tracker/model/impl/EmptyTerminatePayload.java

@@ -0,0 +1,6 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import ru.nbch.pipeline.engine.core.TerminatePayload;
+
+public class EmptyTerminatePayload implements TerminatePayload {
+}

src/main/java/ru/nbch/credit_tracker/model/impl/PackageVariable.java

@@ -0,0 +1,8 @@
+package ru.nbch.credit_tracker.model.impl;
+
+public enum PackageVariable {
+    packageId,
+    cacheMisses,
+    cacheHits,
+    mode
+}

src/main/java/ru/nbch/credit_tracker/model/impl/PartFilePayload.java

@@ -0,0 +1,38 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import lombok.Getter;
+import ru.nbch.credit_tracker.enums.PoisonType;
+import ru.nbch.pipeline.engine.core.StagePayload;
+
+import java.nio.file.Path;
+
+@Getter
+public class PartFilePayload implements StagePayload {
+    private final Path partPath;
+    private final Long packageId;
+    private final String membercode;
+    private final String packageName;
+    private final PoisonType poison;
+    public static final PartFilePayload POISON = new PartFilePayload(PoisonType.NORMALLY);
+
+    public PartFilePayload(PoisonType poison) {
+        this.poison = poison;
+        this.membercode = null;
+        this.packageName = null;
+        this.partPath = null;
+        this.packageId = null;
+    }
+
+    public PartFilePayload(Path partPath, Long packageId, String membercode, String packageName) {
+        this.partPath = partPath;
+        this.packageId = packageId;
+        this.membercode = membercode;
+        this.packageName = packageName;
+        this.poison = PoisonType.NONE;
+    }
+
+    @Override
+    public boolean isPoison() {
+        return poison == PoisonType.NORMALLY;
+    }
+}

src/main/java/ru/nbch/credit_tracker/model/impl/SignalPayload.java

@@ -0,0 +1,6 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import ru.nbch.pipeline.engine.core.TerminatePayload;
+
+public record SignalPayload(boolean hasFids) implements TerminatePayload {
+}

src/main/java/ru/nbch/credit_tracker/model/impl/SubjectPayload.java

@@ -0,0 +1,76 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import ru.nbch.credit_tracker.enums.PoisonType;
+import ru.nbch.credit_tracker.model.SubjectProfile;
+import ru.nbch.pipeline.engine.core.PipelineContext;
+import ru.nbch.pipeline.engine.core.StagePayload;
+
+import java.util.List;
+
+
+public class SubjectPayload implements StagePayload {
+    private final Long packageId;
+    private final String memberCode;
+    private final String packageName;
+    private final List<SubjectProfile> subjects;
+    private PipelineContext<PackageVariable> ctx;
+    public static final SubjectPayload POISON = new SubjectPayload(null, PoisonType.NORMALLY);
+    private final PoisonType poison;
+
+    private boolean hasFids = false;
+
+    public SubjectPayload(Long packageId, PoisonType poison) {
+        this.memberCode = null;
+        this.packageName = null;
+        this.packageId = packageId;
+        this.subjects = null;
+        this.poison = poison;
+    }
+
+    public SubjectPayload(Long packageId, String memberCode, String packageName, List<SubjectProfile> subjects) {
+        this.packageId = packageId;
+        this.memberCode = memberCode;
+        this.packageName = packageName;
+        this.subjects = subjects;
+        this.poison = PoisonType.NONE;
+    }
+
+    @Override
+    public void setContext(PipelineContext<?> context) {
+        this.ctx = (PipelineContext<PackageVariable>) context;
+    }
+
+    @Override
+    public PipelineContext<?> getContext() {
+        return this.ctx;
+    }
+
+    public Long getPackageId() {
+        return packageId;
+    }
+
+    public List<SubjectProfile> getSubjects() {
+        return subjects;
+    }
+
+    public String getMemberCode() {
+        return memberCode;
+    }
+
+    public String getPackageName() {
+        return packageName;
+    }
+
+    public boolean isHasFids() {
+        return hasFids;
+    }
+
+    public void setHasFids(boolean hasFids) {
+        this.hasFids = hasFids;
+    }
+
+    @Override
+    public boolean isPoison() {
+        return poison == PoisonType.NORMALLY;
+    }
+}

src/main/java/ru/nbch/credit_tracker/model/impl/VoidPayload.java

@@ -0,0 +1,24 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import lombok.Getter;
+import ru.nbch.credit_tracker.enums.PoisonType;
+import ru.nbch.pipeline.engine.core.StagePayload;
+
+@Getter
+public class VoidPayload implements StagePayload {
+    private final PoisonType poison;
+    public static final VoidPayload POISON = new VoidPayload(PoisonType.NORMALLY);
+
+    public VoidPayload(PoisonType poison) {
+        this.poison = poison;
+    }
+
+    public VoidPayload() {
+        this.poison = PoisonType.NONE;
+    }
+
+    @Override
+    public boolean isPoison() {
+        return poison == PoisonType.NORMALLY;
+    }
+}

src/main/java/ru/nbch/credit_tracker/refactoring/Suspicios.java

@@ -0,0 +1,5 @@
+package ru.nbch.credit_tracker.refactoring;
+
+public @interface Suspicios {
+    String value();
+}

src/main/java/ru/nbch/credit_tracker/service/DbVersionValidationService.java

@@ -0,0 +1,62 @@
+package ru.nbch.credit_tracker.service;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.AppVersion;
+import ru.nbch.credit_tracker.mapper.signal.AppVersionMapper;
+
+@Service
+public class DbVersionValidationService implements InitializingBean {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    private final AppVersionMapper appVersionMapper;
+    private final CreditTrackerSettings settings;
+    private final String expectedDbVersion = "2.22";
+
+    @Autowired
+    public DbVersionValidationService(AppVersionMapper appVersionMapper, CreditTrackerSettings settings) {
+        this.appVersionMapper = appVersionMapper;
+        this.settings = settings;
+    }
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        String[] expectedVersions = expectedDbVersion.split("\\.");
+        String expectedMajor = expectedVersions[0];
+        String expectedMinor = expectedVersions[1];
+        if (StringUtils.isEmpty(expectedMajor) || StringUtils.isEmpty(expectedMinor)) {
+            log.error("expected database version is not specified in the settings.");
+            System.exit(1);
+        }
+
+        log.info("expected DB version: {}", expectedDbVersion);
+
+        AppVersion appVersion = appVersionMapper.getAppVersion();
+        String[] actualVersions = appVersion.getDbVersion().split("\\.");
+        String actualMajor = actualVersions[0];
+        String actualMinor = actualVersions[1];
+        if (StringUtils.isEmpty(actualMajor) || StringUtils.isEmpty(actualMinor)) {
+            log.error("couldn't load version from APP_VERSION database table.");
+            System.exit(1);
+        }
+
+        log.info("actual DB version: {}", appVersion.getDbVersion());
+
+        if (!expectedMajor.equals(actualMajor)) {
+            log.error("expected database version does not match actual database version");
+            System.exit(1);
+        }
+
+        if (Integer.parseInt(expectedMinor) < Integer.parseInt(actualMinor)) {
+            log.error("expected database version does not match actual database version");
+            System.exit(1);
+        }
+
+        log.info("Database version validation complete: {}", appVersion.getDbVersion());
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/MonitoringErrorProcesingService.java

@@ -1,59 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.aop.framework.AopContext;
-import org.springframework.context.annotation.EnableAspectJAutoProxy;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.mdc.SetAndClearMdc;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-import ru.nbch.credit_tracker.utils.CTUtil;
-
-@Slf4j
-@Service
-@EnableAspectJAutoProxy(exposeProxy = true)
-public class MonitoringErrorProcesingService {
-
-    private MonitoringErrorProcesingService self;
-
-    private final SignalService signalService;
-    private final SignalRestService signalRestService;
-
-    public MonitoringErrorProcesingService(SignalService signalService, SignalRestService signalRestService) {
-        this.signalService = signalService;
-        this.signalRestService = signalRestService;
-    }
-
-    public void findErrors(Long packageId) {
-        UserPackage userPackage = signalService.findPackage(packageId);
-        getCurrentProxy().processPackage(userPackage, packageId);
-    }
-
-    @SetAndClearMdc
-    public void processPackage(UserPackage userPackage, Long packageId) {
-        if (userPackage != null) {
-            String packCode = userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName();
-            log.info("Проверка на ошибки при постановке на мониторинг. packCode={}, packageId={}", packCode, packageId);
-            String fileName = signalRestService.getErrorFileName(packageId, packCode);
-            if (StringUtils.isNotBlank(fileName)) {
-                String filePath = signalRestService.loadErrorFile(packageId, fileName);
-                if (StringUtils.isNotBlank(filePath)) {
-                    try {
-                        signalService.updateSubjectsErrors(filePath, packageId);
-                    } catch (Exception e) {
-                        log.error("Ошибка при записи сообщений об ошибках в сформированных субъектах для мониторинга. Package_id = {}", packageId, e);
-                    }
-                    CTUtil.removeFile(filePath);
-                }
-            }
-        }
-    }
-
-    private MonitoringErrorProcesingService getCurrentProxy() {
-        if (self == null) {
-            self = (MonitoringErrorProcesingService) AopContext.currentProxy();
-        }
-        return self;
-    }
-}

src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java

@@ -1,319 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.MDC;
-import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.Hit;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.entities.external_response.online_download.Inquiry;
-import ru.nbch.credit_tracker.entities.external_response.online_download.Person;
-import ru.nbch.credit_tracker.entities.external_response.online_download.Persons;
-import ru.nbch.credit_tracker.entities.external_response.online_report.Report;
-import ru.nbch.credit_tracker.entities.external_response.online_report.SgnlOnlineReport;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.MonitoringSignalResponse;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Signal;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Signals;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Subject;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlWriter;
-import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
-import ru.nbch.credit_tracker.utils.CTUtil;
-
-import java.io.*;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.LocalDateTime;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-/**
- * Сервис периодического опроса официального сервиса сигналов на наличие новых отчетов по пакетам находящимся в статусе ON_MONITORING
- */
-@Slf4j
-@Service
-public class MonitoringService {
-
-    private final SignalRestService signalRestService;
-    private final SignalService signalService;
-    private final StaxXmlProcessor staxXmlProcessor;
-    private final CreditTrackerSettings config;
-
-    public MonitoringService(SignalRestService signalRestService,
-                             SignalService signalService,
-                             StaxXmlProcessor staxXmlProcessor,
-                             CreditTrackerSettings config) {
-        this.signalRestService = signalRestService;
-        this.signalService = signalService;
-        this.staxXmlProcessor = staxXmlProcessor;
-        this.config = config;
-    }
-
-    public void run() {
-        ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-"));
-        long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15;
-        scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES);
-    }
-
-    private void monitor() {
-        log.info("Start monitoring");
-        List<UserPackage> packages = signalService.findPackagesOnMonitoring();
-        for (UserPackage userPackage : packages) {
-            MDC.put("ru.nbch.credit_tracker.log.mdc.key", "[" + userPackage.getPackageName() +"]");
-            processPackage(userPackage);
-            MDC.remove("ru.nbch.credit_tracker.log.mdc.key");
-        }
-        log.info("Finish monitoring");
-    }
-
-    private void processPackage(UserPackage userPackage) {
-        log.info("Processing package");
-        if (!signalService.hasNonNullfids(userPackage.getId())) {
-            log.info("Package has no fids. Package will be skipped.");
-            return;
-        }
-        SgnlOnlineReport onlineReports = signalRestService.getReports(userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName());
-
-        if (onlineReports == null) {
-            log.info("No reports found for package");
-            return;
-        }
-
-        // Формируем список имен файлов, по которым сработали сигналы. Сюда входят только те имена, у которых парсится дата формирования отчета
-        List<String> fileNames = onlineReports.getOperationReport().getReports().getReports().stream().map(Report::getFilename)
-                .filter(fileName -> CTUtil.parseDateFromReportFileName(fileName) != null).toList();
-
-        if (fileNames.isEmpty()) {
-            log.info("No new report found for package");
-            return;
-        }
-
-        LocalDateTime lastDownloadedReportDate = CTUtil.parseDateFromReportFileName(userPackage.getLastDownloadedReport());
-
-        if (lastDownloadedReportDate != null) {
-            // Отбираем только те отчеты, дата которых позже даты последнего загруженного отчета
-            fileNames = fileNames.stream()
-                    .filter(fileName -> CTUtil.parseDateFromReportFileName(fileName).isAfter(lastDownloadedReportDate)) // отфильтровали только те имена, которые парсятся в дату, так что null тут не придет
-                    .sorted(Comparator.comparing(CTUtil::parseDateFromReportFileName)) // Сортируем по дате
-                    .toList();
-        } else {
-            fileNames = fileNames.stream()
-                    .sorted(Comparator.comparing(CTUtil::parseDateFromReportFileName)) // Сортируем по дате
-                    .toList();
-        }
-
-        if (StringUtils.isNotBlank(userPackage.getLastDownloadedReport())) {
-            log.info("Last downloaded report found for package. Last Downloaded Report Name: {}", userPackage.getLastDownloadedReport());
-        } else {
-            log.info("Last Downloaded Report Name is empty");
-        }
-
-        for (String newReport : fileNames) {
-            try {
-                String loadedReportPath = null;
-                if (StringUtils.isBlank(userPackage.getLastDownloadedReport())) {
-                    loadedReportPath = loadReport(userPackage.getId(), newReport); // по этому пакету еще не было загруженных отчетов, продолжаем работу с полученным
-                } else if (lastDownloadedReportDate != null) {
-                    LocalDateTime newReportDate = CTUtil.parseDateFromReportFileName(newReport);
-                    if (newReportDate.isAfter(lastDownloadedReportDate)) {
-                        loadedReportPath = loadReport(userPackage.getId(), newReport); // полученный отчет новее записанного в базу
-                    }
-                }
-                if (loadedReportPath == null) {
-                    log.info("Report was not loaded for package");
-                    continue;
-                }
-
-                String responsePath = generateResponse(userPackage, loadedReportPath);
-                String fileName = null;
-                if (responsePath != null) {
-                    fileName = createResponseFile(userPackage, responsePath);
-                }
-                if (fileName != null) {
-                    log.info("New report fileName: {}", fileName);
-                    updateSignalHits(userPackage, responsePath, fileName);
-                    signalService.updateLastDownloadedReport(userPackage.getId(), newReport);
-                } else {
-                    log.info("Report was not created for package");
-                }
-            } catch (Exception e) {
-                log.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    /**
-     * Формирует результирующий xml файл для конечного пользователя на основе файла из официального сервиса сигналов
-     * @return Путь к временному рещультирующему файлу
-     */
-    private String generateResponse(UserPackage userPackage, String loadedReportPath) {
-        AtomicBoolean generateResponse = new AtomicBoolean(false); // Выставляется в true, если хотя бы один субъект с мониторинга подошел под условие срабатывания сигнала. В этом случае отчет сохраняется
-
-        String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator;
-        String responsePath = folderPath + UUID.randomUUID() + "_response_output.xml";
-        CTUtil.createFolder(folderPath);
-
-        MonitoringSignalResponse response = new MonitoringSignalResponse();
-        response.setPackageName(userPackage.getPackageName()); // Имя пакета, переданного при установке
-
-        // Запись в результрующий файл
-        MonitoringSignalResponseStaxXmlWriter staxXmlWriter = new MonitoringSignalResponseStaxXmlWriter(config.getEncoding());
-        staxXmlWriter.writeWithStax(responsePath, response,
-                writer -> {
-                    try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(loadedReportPath)), 8 * 1024 * 1024)) {
-                        staxXmlProcessor.read(in, Persons.class,
-                                personsData -> { // как только считали пачку Persons сразу записали ее в результирующий файл
-                                    List<Subject> subjectsBatch = processPersons(personsData.getPersons());
-                                    if (!subjectsBatch.isEmpty()) {
-                                        generateResponse.set(true);
-                                    }
-                                    writer.accept(subjectsBatch);
-                                    personsData.getPersons().clear(); // чистим список для экономии памяти
-                                },
-                                pesons -> pesons.getPersons().isEmpty()
-                                ,
-                                10000,
-                                "/Report/Persons/Person");
-                    } catch (IOException e) {
-                        throw new RuntimeException("Error while parsing nbki downloaded report", e);
-                    }
-                });
-
-
-        CTUtil.removeFile(loadedReportPath);
-
-        if (!generateResponse.get()) { // ни один субъект не подошел под условие, удаляем временный отчет
-            CTUtil.removeFile(responsePath);
-            return null;
-        }
-
-        return responsePath;
-    }
-
-    private List<Subject> processPersons(List<Person> persons) {
-        List<Subject> subjects = new ArrayList<>();
-        for (Person person : persons) {
-            Signals signals = new Signals();
-            for (Inquiry inquiry : person.getInquiries()) {
-                // Отбираем только тех фл, у которых bus_category in (MFO, MKK, MFK) и есть сигналы с кодом 16
-                if (!CTUtil.checkBusCategory(inquiry.getBusCategory()) || inquiry.getSignals().stream().noneMatch(signal -> signal.getN().equals(16))) {
-                    continue;
-                }
-                Signal signal = new Signal();
-                signal.setSignalType(1); // Код типа сигнала по внутреннему справочнику. В текущей версии: 1 = сигнал 16 от MFO/MKK/MFK
-                signal.setInquiryDate(inquiry.getDate()); // Дата и время запроса КИ
-                signal.setInquiryPurpose(inquiry.getInqP()); // Цель запроса КИ
-                signals.getSignals().add(signal);
-            }
-
-            // Если есть хотя бы один заполненный сигнал, значит заполняем остальные данные
-            if (!signals.getSignals().isEmpty()) {
-                PhoneFidMap subjectData = signalService.findPhoneFidMapById(person.getUid());
-                if (subjectData != null) {
-                    Subject subject = new Subject();
-                    subject.setId(subjectData.getSubjectId()); // Уникальный идентификатор субъекта (равен Id из XML-пакета при установке мониторинга)
-                    subject.setClientId(subjectData.getPhone());
-                    subject.setFlagDebt5000(subjectData.getFlagDebt5000());
-                    subject.setFlag9012(subjectData.getFlag90days());
-                    subject.setSignals(signals);
-                    subjects.add(subject);
-                }
-            }
-        }
-        return subjects;
-    }
-
-    /**
-     * Архивирует результирующий xml файл для пользователя
-     * @return Путь к результрующему файлу. Пример <loadDir>/<member_code>/0001ZZ_MY_BATCH_TEST_0717_04_20250723_101929.zip
-     */
-    private String createResponseFile(UserPackage userPackage, String responsePath) {
-        try {
-            String fileName = CTUtil.generateResponseReportFileName(userPackage.getUserId(), userPackage.getPackageName());
-            Thread.sleep(1000); // Если отчеты формируются слишком быстро, нужно подождать секунду, тк в имени отчета LocalDateTime до секунд
-            String folderPath = config.getLoadDir() + File.separator + userPackage.getUserId().substring(0, 6) + File.separator;
-
-            File folder = new File(folderPath);
-            if (!folder.exists()) {
-                folder.mkdir();
-            }
-
-            try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(responsePath)), 8 * 1024 * 1024);
-                 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(folderPath + fileName + ".zip"));
-                 ZipOutputStream zos = new ZipOutputStream(bos)) {
-
-                ZipEntry zipEntry = new ZipEntry(fileName + ".xml");
-                zos.putNextEntry(zipEntry);
-                byte[] buffer = new byte[8 * 1024 * 1024];
-                int bytesRead;
-                while ((bytesRead = in.read(buffer)) > 0) {
-                    zos.write(buffer, 0, bytesRead);
-                }
-                zos.finish();
-                zos.closeEntry();
-            }
-            return fileName;
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-        } catch (Exception e) {
-            log.error("Ошибка при записи отчета пользователя в файл: ", e);
-        }
-
-        CTUtil.removeFile(responsePath);
-
-        return null;
-    }
-
-    private void updateSignalHits(UserPackage userPackage, String responsePath, String reportFileName) {
-        log.info("Updating signal hits for {}", userPackage.getPackageName());
-        LocalDateTime responseSentAt = LocalDateTime.now();
-        // Считываем результирующий файл пачками и обновляем таблицу hits
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(responsePath)), 8 * 1024 * 1024)) {
-            staxXmlProcessor.read(in, MonitoringSignalResponse.class,
-                    response -> { // как только считали пачку Subjects сразу обновлям таблицу hits
-                        List<Hit> hits = new ArrayList<>();
-                        response.getSubjects().getSubjects().forEach(subject -> {
-                            subject.getSignals().getSignals().forEach(signal -> {
-                                Hit hit = new Hit();
-                                hit.setPackageId(userPackage.getId());
-                                hit.setSubjectId(subject.getId());
-                                hit.setSignalType(1);
-                                hit.setResponseSentAt(responseSentAt);
-                                hit.setReportFileName(reportFileName);
-                                hits.add(hit);
-                            });
-                            if (!hits.isEmpty()) {
-                                signalService.insertHits(hits);
-                            }
-                        });
-                        response.getSubjects().getSubjects().clear(); // чистим список для экономии памяти
-                    },
-                    response -> response.getSubjects().getSubjects().isEmpty()
-                    ,
-                    10000,
-                    "/MonitoringSignalResponse/Subjects/Subject");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while updating hits", e);
-        }
-
-        CTUtil.removeFile(responsePath); // удалям результирующий xml файл с диска (Он уже заархивирован к этому моменту)
-
-        log.info("Signal hits updated for package {}", userPackage.getPackageName());
-    }
-
-    public String loadReport(Long packageId, String reportName) {
-        return signalRestService.downloadReport(packageId, reportName);
-    }
-
-}

src/main/java/ru/nbch/credit_tracker/service/SignalService.java

@@ -1,366 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.builder.PersonFactoryBuilder;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.Hit;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.entities.external_request.signals.Auth;
-import ru.nbch.credit_tracker.entities.external_request.signals.Person;
-import ru.nbch.credit_tracker.entities.external_request.signals.Signals;
-import ru.nbch.credit_tracker.entities.external_response.monitoring_report_error.MonitoringReportErrors;
-import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
-import ru.nbch.credit_tracker.enums.Status;
-import ru.nbch.credit_tracker.service.indic.FlagsService;
-import ru.nbch.credit_tracker.service.indic.PersonService;
-import ru.nbch.credit_tracker.service.indic.PhoneService;
-import ru.nbch.credit_tracker.service.signal.HitsService;
-import ru.nbch.credit_tracker.service.signal.PhoneFidMapService;
-import ru.nbch.credit_tracker.service.signal.UserPackageService;
-import ru.nbch.credit_tracker.service.xml.stax.SignalsStaxXmlWriter;
-import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
-import ru.nbch.credit_tracker.utils.CTUtil;
-
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.LocalDateTime;
-import java.util.*;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-
-@Slf4j
-@Service
-public class SignalService {
-
-    private final CreditTrackerSettings config;
-    private final HitsService hitsService;
-    private final UserPackageService userPackageService;
-    private final PhoneFidMapService phoneFidMapService;
-    private final PhoneService phoneService;
-    private final FlagsService flagsService;
-    private final PersonService personService;
-    private final StaxXmlProcessor staxXmlProcessor;
-
-    private final int batchSize = 10000;
-
-    public SignalService(CreditTrackerSettings config,
-                         HitsService hitsService,
-                         UserPackageService userPackageService,
-                         PhoneFidMapService phoneFidMapService,
-                         PhoneService phoneService,
-                         FlagsService flagsService,
-                         PersonService personService,
-                         StaxXmlProcessor staxXmlProcessor) {
-        this.config = config;
-        this.hitsService = hitsService;
-        this.userPackageService = userPackageService;
-        this.phoneFidMapService = phoneFidMapService;
-        this.phoneService = phoneService;
-        this.flagsService = flagsService;
-        this.personService = personService;
-        this.staxXmlProcessor = staxXmlProcessor;
-    }
-
-    public boolean isPackageNotUnique(MonitoringRequest request) {
-        return userPackageService.isPackageNotUnique(request.getUserId(), request.getPackageName());
-    }
-
-    public Long registerPackage(MonitoringRequest request) {
-        UserPackage userPackage = UserPackage.builder()
-                .userId(request.getUserId())
-                .packageName(request.getPackageName())
-                .createdAt(LocalDateTime.now())
-                .status(Status.PROCESSING.name())
-                .build();
-        return userPackageService.insertPackage(userPackage);
-    }
-
-    public void registerSubjects(String filePath, Long packageId) {
-        log.info("Start registering subjects. PackageId: {}", packageId);
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(filePath)), 8 * 1024 * 1024)) {
-            staxXmlProcessor.read(in, MonitoringRequest.class,
-                    request -> {
-                        if (!request.getSubjects().getSubject().isEmpty()) {
-                            phoneFidMapService.registerSubjects(request.getSubjects().getSubject(), packageId);
-                            log.trace("Registered subject's batch. Total count: {}. PackageId: {}", request.getSubjects().getSubject().size(), packageId);
-                            request.getSubjects().getSubject().clear();
-                        }
-                    },
-                    request -> request.getSubjects().getSubject().isEmpty()
-                    ,
-                    batchSize, "/MonitoringRequest/Subjects/Subject");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while parsing monitoring request", e);
-        }
-        log.info("Subjects were registered. PackageId: {}", packageId);
-    }
-
-    /**
-     * Обрабатывает все PhoneFidMap по packageId, выгружая их пачками и фильтруя по переданному фильтру
-     */
-    public UpdateResult updateSubjects(Long packageId, Predicate<PhoneFidMap> filter) {
-        boolean updated = false;
-        boolean noFidsFound = true;
-        int offset = 0;
-        List<PhoneFidMap> batch;
-        do {
-            batch = phoneFidMapService.findFullPhoneFidMap(packageId, offset, batchSize);
-
-            List<PhoneFidMap> filteredItems;
-            if (filter != null) {
-                filteredItems = batch.stream().filter(filter).toList();
-            } else {
-                filteredItems = batch;
-            }
-
-            if (!updated && !filteredItems.isEmpty()) {
-                updated = true;
-            }
-
-            if (!filteredItems.isEmpty()) {
-                boolean fidsFound = updateFlags(filteredItems, packageId);
-                if (noFidsFound) {
-                    noFidsFound = !fidsFound;
-                }
-            }
-
-            offset += batchSize;
-        } while (!batch.isEmpty());
-
-        return new UpdateResult(updated, noFidsFound);
-    }
-
-    /**
-     * Обрабатывает пачку PhoneFidMap: обновляет фиды и флаги просрочек
-     * @return fidsFound true - если хотя бы по одному номеру был найден фид, false - в противном случае
-     */
-    public boolean updateFlags(List<PhoneFidMap> subjects, Long packageId) {
-        log.info("Start calculating fids and flags for subjects batch. PackageId: {}", packageId);
-        Map<String, Integer> fidsMap = phoneService.findFidsPhoneData(subjects);
-        LocalDateTime calculatedAt = LocalDateTime.now();
-        boolean fidsFound = fidsMap.values().stream().anyMatch(Objects::nonNull);
-        Map<Integer, FlagsService.FlagsData> flagsDataMap = flagsService.findFlags(fidsMap.values().stream().filter(Objects::nonNull).toList());
-        List<PhoneFidMap> subjectToUpdate = new ArrayList<>();
-
-        subjects.forEach(subject -> {
-            Integer fid = fidsMap.get("9" + subject.getPhone());
-            if (fid != null) {
-                FlagsService.FlagsData flagsData = flagsDataMap.get(fid);
-                subject = PhoneFidMap.builder()
-                        .id(subject.getId())
-                        .packageId(packageId)
-                        .subjectId(subject.getSubjectId())
-                        .phone(subject.getPhone())
-                        .fid(fid)
-                        .flagDebt5000(flagsData.getFlagDebt5000())
-                        .flag90days(flagsData.getFlag90days())
-                        .calculatedAt(calculatedAt)
-                        .build();
-            } else {
-                subject = PhoneFidMap.builder()
-                        .id(subject.getId())
-                        .packageId(packageId)
-                        .subjectId(subject.getSubjectId())
-                        .phone(subject.getPhone())
-                        .calculatedAt(calculatedAt)
-                        .build();
-            }
-            subjectToUpdate.add(subject);
-        });
-
-        int updateBatchSize = batchSize / 9; // 9 параметров на одну запись для обновления. Максимальное количество параметров ~65000
-        for (int j = 0; j < subjectToUpdate.size(); j += updateBatchSize) {
-            int endIndex = Math.min(j + updateBatchSize, subjectToUpdate.size());
-            phoneFidMapService.updatePhoneFidMaps(subjectToUpdate.subList(j, endIndex));
-            log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjectToUpdate.size(), packageId);
-        }
-
-        log.info("Finish calculating fids and flags for subjects. PackageId: {}. Batch size: {}", packageId, subjects.size());
-
-        return fidsFound;
-    }
-
-    public void rejectPackage(MonitoringRequest request) {
-        UserPackage userPackage = UserPackage.builder()
-                .userId(request.getUserId())
-                .packageName(request.getPackageName())
-                .createdAt(LocalDateTime.now())
-                .status(Status.REJECTED.name())
-                .build();
-        userPackageService.insertPackage(userPackage);
-    }
-
-    public void updatePackageStatus(Long packageId, Status status) {
-        userPackageService.updatePackageStatus(packageId, status);
-    }
-
-    public void updatePackageOnMonitoringSuccess(Long packageId, LocalDateTime monitoringAt) {
-        userPackageService.updatePackageOnMonitoringSuccess(packageId, monitoringAt);
-    }
-
-    /**
-     * Формирует файл запроса в официальный сервис сигналов, выгружая паспортные данные пачками и сразу записывая их в файл
-     *
-     * @return Путь к файлу запроса к официальному сервису сигналов на постановку на мониторинг
-     */
-    public String writeSignals(Long packageId) {
-        String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator;
-        String filePath = folderPath + UUID.randomUUID() + "_output.xml";
-        CTUtil.createFolder(folderPath);
-
-        Signals signals = buildSignals(packageId);
-
-        SignalsStaxXmlWriter signalsStaxXmlWriter = new SignalsStaxXmlWriter(config.getEncoding());
-
-        signalsStaxXmlWriter.writeWithStax(filePath,
-                signals,
-                writer -> buildPersons(packageId, writer));
-
-        return filePath;
-    }
-
-    private Signals buildSignals(Long packageId) {
-        UserPackage userPackage = userPackageService.findPackage(packageId);
-        Signals signals = new Signals();
-        signals.setOnline("2"); // Всегда=2 (только онлайн-сигналы)
-        signals.setProcessWithErrors("1"); // Всегда=1 (обработать пакет полностью, по обнаруженным ошибкам (при наличии) сформировать файл отказа, записи без ошибок установить на мониторинг)
-        signals.setId(userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName()); // <MEMBERCODE>_<PackageName>
-        Auth auth = new Auth();
-        auth.setUser(config.getSignalsuser());
-        signals.setAuth(auth);
-        return signals;
-    }
-
-    /**
-     * Обрабатывает все PhoneFidMap по packageId, выгружая их пачками и фильтруя по переданному фильтру.
-     * writer записывает пачку сразу в файл
-     */
-    private void buildPersons(Long packageId, Consumer<List<Person>> writer) {
-        int offset = 0;
-        List<PhoneFidMap> batch;
-        do {
-            batch = phoneFidMapService.findFullPhoneFidMap(packageId, offset, batchSize);
-            List<Person> persons = findPersons(batch);
-            if (!persons.isEmpty()) {
-                writer.accept(persons); // записывает пачку Persons в файл
-            }
-            offset += batchSize;
-        } while (!batch.isEmpty());
-
-    }
-
-    /**
-     * Формирует список Person для постановки на мониторинг в официальный сервис сигналов на основе списка PhoneFidMap
-     */
-    private List<Person> findPersons(List<PhoneFidMap> subjects) {
-        if (subjects.isEmpty()) {
-            return Collections.emptyList();
-        }
-        List<Person> persons = new ArrayList<>();
-        Map<Integer, PersonService.PersonData> personFidMap = personService.findPersonData(subjects.stream()
-                .map(PhoneFidMap::getFid)
-                .filter(Objects::nonNull).toList());
-        subjects.forEach(data -> {
-            if (data.getFid() != null) {
-                PersonService.PersonData personData = personFidMap.get(data.getFid());
-                Person person = PersonFactoryBuilder.builder()
-                        .personData(personData)
-                        .uid(data.getId())
-                        .build();
-                if (person != null) {
-                    persons.add(person);
-                }
-            }
-        });
-        return persons;
-    }
-
-    public void updateSubjectsErrors(String filePath, Long packageId) {
-        log.info("Start updating subjects errors. PackageId: {}", packageId);
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(filePath)), 8 * 1024 * 1024)) {
-            staxXmlProcessor.read(in, MonitoringReportErrors.class,
-                    request -> {
-                        if (!request.getPersons().isEmpty()) {
-                            List<PhoneFidMap> subjects = request.getPersons().stream().map(p -> {
-                                log.warn("Ошибка при формировании субъекта. package_id = {}, id = {}. Описание: {}", packageId, p.getUid(), p.getValue());
-                                return PhoneFidMap.builder()
-                                        .id(p.getUid())
-                                        .errorText(p.getValue())
-                                        .build();
-                            }).toList();
-                            phoneFidMapService.updatePhoneFidMapsErrors(subjects);
-                            log.trace("Updated subject's batch. Total count: {}. PackageId: {}", request.getPersons().size(), packageId);
-                            request.getPersons().clear();
-                        }
-                    },
-                    request -> request.getPersons().isEmpty()
-                    ,
-                    batchSize, "/Errors/p");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while parsing MonitoringReportErrors", e);
-        }
-        log.info("Subject's errors were updated. PackageId: {}", packageId);
-    }
-
-    public List<UserPackage> findActivePackages(String userId) {
-        return userPackageService.findUserPackages(userId, Status.ON_MONITORING);
-    }
-
-    public List<UserPackage> findAllPackages(String userId) {
-        return userPackageService.findAllPackagesByUserId(userId);
-    }
-
-    public boolean isPackageExists(String username, String packageName) {
-        return userPackageService.isPackageExists(username, packageName, Status.ON_MONITORING);
-    }
-
-    public boolean isPackagesByStatusExist(String username, String packageName, Status status) {
-        return userPackageService.hasActivePackages(username, packageName, status);
-    }
-
-    public void deletePackage(String username, String packageName, LocalDateTime deactivatedAt) {
-        userPackageService.deletePackage(username, packageName, Status.DELETED, Status.ON_MONITORING, deactivatedAt);
-    }
-
-    public List<UserPackage> findPackagesOnMonitoring() {
-        return userPackageService.findAllPackagesByStatus(Status.ON_MONITORING);
-    }
-
-    public List<UserPackage> findPackagesOnProcessing() {
-        return userPackageService.findAllPackagesByStatus(Status.PROCESSING);
-    }
-
-    public void insertHits(List<Hit> hits) {
-        hitsService.insertHits(hits);
-    }
-
-    public void updateLastDownloadedReport(Long id, String fileName) {
-        userPackageService.updatePackageLastDownloadedReport(id, fileName);
-    }
-
-    public PhoneFidMap findPhoneFidMapById(Long id) {
-        return phoneFidMapService.findPhoneFidMapById(id);
-    }
-
-    public boolean hasNonNullfids(Long packageId) {
-        return phoneFidMapService.hasNonNullfids(packageId);
-    }
-
-    public UserPackage findPackage(Long packageId) {
-        return userPackageService.findPackage(packageId);
-    }
-
-    @Getter
-    @AllArgsConstructor
-    public static class UpdateResult {
-        private boolean updated;
-        private boolean noFidsFound;
-    }
-}

src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java

@@ -1,76 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.aop.framework.AopContext;
-import org.springframework.context.annotation.EnableAspectJAutoProxy;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.mdc.SetAndClearMdc;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.enums.Status;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-
-import java.util.List;
-
-/**
- * Сервис для завершения обработки пакетов находящихся в статусе PROCESSING
- * Запускается при старте приложения и отрабатывает 1 раз
- */
-@Slf4j
-@Service
-@EnableAspectJAutoProxy(exposeProxy = true)
-public class UpdateProcessingPackagesService {
-
-    private UpdateProcessingPackagesService self;
-
-    private final SignalService signalService;
-    private final SignalRestService signalRestService;
-    private final MonitoringErrorProcesingService monitoringErrorProcesingService;
-
-    public UpdateProcessingPackagesService(SignalService signalService,
-                                           SignalRestService signalRestService,
-                                           MonitoringErrorProcesingService monitoringErrorProcesingService) {
-        this.signalService = signalService;
-        this.signalRestService = signalRestService;
-        this.monitoringErrorProcesingService = monitoringErrorProcesingService;
-    }
-
-    public void run() {
-        log.info("Start updating processing packages");
-        List<UserPackage> packages = signalService.findPackagesOnProcessing();
-        for (UserPackage userPackage : packages) {
-            log.info("Updating flags for Package {}", userPackage.getPackageName());
-            getCurrentProxy().process(userPackage.getId(), userPackage.getPackageName());
-        }
-        log.info("Finish updating processing packages");
-    }
-
-    @SetAndClearMdc(argIndex = 1)
-    public void process(Long packageId, String packageName) {
-        try {
-            SignalService.UpdateResult updateResult = signalService.updateSubjects(packageId, null);
-            if (updateResult.isNoFidsFound()) {
-                log.info("No fids found for PackageId: {} . Package status ON_MONITORING. No registration in signal service.", packageId);
-                signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-                return;
-            }
-            if (signalRestService.setupMonitoring(packageId)) {
-                log.info("Package was successfully registered in signal service. PackageId: {}", packageId);
-                signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-                monitoringErrorProcesingService.findErrors(packageId);
-            } else {
-                log.info("Package registration was rejected by signal service. PackageId: {}", packageId);
-                signalService.updatePackageStatus(packageId, Status.ERROR);
-            }
-        } catch (Exception e) {
-            signalService.updatePackageStatus(packageId, Status.ERROR);
-            log.error("Ошибка при поиске фидов или расчете флагов просрочек. Package_id {}", packageId, e);
-        }
-    }
-
-    private UpdateProcessingPackagesService getCurrentProxy() {
-        if (self == null) {
-            self = (UpdateProcessingPackagesService) AopContext.currentProxy();
-        }
-        return self;
-    }
-}

src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java

@@ -1,94 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.extern.slf4j.Slf4j;
-import org.slf4j.MDC;
-import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.enums.Status;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-
-import java.time.LocalDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Predicate;
-
-/**
- * Сервис периодического обновления фидов и флагов просрочек по пакетам находящимся в статусе ON_MONITORING
- * По обновленным пакетам формируется новый запрос постановки на мониторинг в официальный сервис сигналов
- */
-@Slf4j
-@Service
-public class UpdatingFlagsService {
-
-    private final SignalService signalService;
-    private final SignalRestService signalRestService;
-    private final MonitoringErrorProcesingService monitoringErrorProcesingService;
-    private final Long daysToUpdateFlags;
-    private final Long updateFlagsInterval;
-
-    public UpdatingFlagsService(SignalService signalService,
-                                SignalRestService signalRestService,
-                                MonitoringErrorProcesingService monitoringErrorProcesingService,
-                                CreditTrackerSettings config) {
-        this.signalService = signalService;
-        this.signalRestService = signalRestService;
-        this.monitoringErrorProcesingService = monitoringErrorProcesingService;
-        this.daysToUpdateFlags = config.getDaysToUpdateFlags() != null ? config.getDaysToUpdateFlags() : 1;
-        this.updateFlagsInterval = config.getUpdateFlagsInterval() != null ? config.getUpdateFlagsInterval() : 1;
-    }
-
-    public void run() {
-        ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("updating-flags-"));
-        scheduler.scheduleAtFixedRate(this::update, 0, updateFlagsInterval, TimeUnit.DAYS);
-    }
-
-    private void update() {
-        log.info("Start updating flags");
-        List<UserPackage> packages = signalService.findPackagesOnMonitoring();
-        for (UserPackage userPackage : packages) {
-            log.info("Updating flags for Package {}", userPackage.getPackageName());
-            MDC.put("ru.nbch.credit_tracker.log.mdc.key", "[" + userPackage.getPackageName() +"]");
-            updateFlags(userPackage);
-            MDC.remove("ru.nbch.credit_tracker.log.mdc.key");
-        }
-        log.info("Finish updating flags");
-    }
-
-    private void updateFlags(UserPackage userPackage) {
-        Long packageId = userPackage.getId();
-        String packageName = userPackage.getPackageName();
-        try {
-            signalService.updatePackageStatus(packageId, Status.PROCESSING);
-            Predicate<PhoneFidMap> filter = subject -> subject.getCalculatedAt() != null && hasDaysPassed(subject.getCalculatedAt());
-            SignalService.UpdateResult updateResult = signalService.updateSubjects(packageId, filter);
-            if (updateResult.isUpdated()) {
-                if (signalRestService.setupMonitoring(packageId)) {
-                    LocalDateTime monitoringAt = userPackage.getMonitoringAt() != null ? userPackage.getMonitoringAt() : LocalDateTime.now();
-                    log.info("Flags were updated successfully. PackageId: {} PackageName: {} MonitoringAt: {}. Package status changed to ON_MONITORING", packageId, packageName, monitoringAt);
-                    signalService.updatePackageOnMonitoringSuccess(packageId, monitoringAt);
-                    monitoringErrorProcesingService.findErrors(packageId);
-                } else {
-                    log.info("Updated package registration was rejected by signal service. PackageId: {} PackageName: {}", packageId, packageName);
-                    signalService.updatePackageStatus(packageId, Status.ERROR);
-                }
-            } else {
-                log.info("No flags were updated for package id {}. Package status changed to ON_MONITORING", packageId);
-                signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-            }
-        } catch (Exception e) {
-            log.error("Ошибка при поиске фидов или перерасчете флагов просрочек. Package_id {}", packageId, e);
-        }
-    }
-
-    private boolean hasDaysPassed(LocalDateTime pastDateTime) {
-        LocalDateTime now = LocalDateTime.now();
-        long daysPassed = ChronoUnit.DAYS.between(pastDateTime, now);
-        return daysPassed >= daysToUpdateFlags;
-    }
-}