image to nexus

http://jira.mfd.msk:8088/browse/BKI-3241

.gitignore

@@ -31,3 +31,4 @@ build/
 
 ### VS Code ###
 .vscode/
+tmp/

Dockerfile

@@ -1,4 +1,4 @@
-FROM eclipse-temurin:17-jre-alpine
+FROM eclipse-temurin:21-jre-alpine
 MAINTAINER ru.nbch
 
 VOLUME /opt/app/logs/app
@@ -8,7 +8,8 @@ VOLUME /opt/app/reports
 
 WORKDIR /opt/app
 COPY maven/elastic-apm-agent-1.45.0.jar /opt/app/elastic-apm-agent.jar
-COPY maven/credit-tracker.jar /opt/app/credit-tracker.jar
+COPY maven/credit-tracker-main.jar /opt/app/credit-tracker-main.jar
+COPY maven/credit-tracker-monitoring.jar /opt/app/credit-tracker-monitoring.jar
 COPY maven/start-credit-tracker.sh /opt/app/start-credit-tracker.sh
 COPY maven/credit-tracker.properties /opt/app/credit-tracker.properties
 COPY maven/monitoring-request.xsd /opt/app/monitoring-request.xsd
@@ -20,4 +21,3 @@ RUN apk add --no-cache ca-certificates \
 RUN apk add gcompat bash
 
 RUN chmod +x /opt/app/start-credit-tracker.sh
-ENTRYPOINT ["/opt/app/start-credit-tracker.sh"]

credit-tracker.properties

@@ -8,10 +8,10 @@ server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b
 # datasources
 # indic
 ct.indic-data-source-settings.driver=com.ibm.db2.jcc.DB2Driver
-ct.indic-data-source-settings.url=${DB_URL_INDIC:jdbc:db2://10.230.227.101:2668/cprosd22:currentSchema=INDIC;}
-ct.indic-data-source-settings.username=${USERNAME_INDIC:tester}
-ct.indic-data-source-settings.password=${PASSWORD_INDIC:tester}
-ct.indic-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_INDIC:10}
+ct.indic-data-source-settings.url=${DB_URL_INDIC:jdbc:db2://10.230.227.102:2668/cprosd22:currentSchema=INDIC;}
+ct.indic-data-source-settings.username=${USERNAME_INDIC:ros9appl}
+ct.indic-data-source-settings.password=${PASSWORD_INDIC:R0s9Appl}
+ct.indic-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_INDIC:30}
 ct.indic-data-source-settings.minimum-idle=${MINIMUM_IDLE_INDIC:10}
 ct.indic-data-source-settings.connection-timeout=${CONNECTION_TIMEOUT_INDIC:10000}
 ct.indic-data-source-settings.validation-timeout=${VALIDATION_TIMEOUT_INDIC:5000}
@@ -22,10 +22,10 @@ ct.indic-data-source-settings.leak-detection-threshold=${LEAK_DETECTION_THRESHOL
 
 # signal
 ct.signal-data-source-settings.driver=org.postgresql.Driver
-ct.signal-data-source-settings.url=${DB_URL_SIGNAL:jdbc:postgresql://10.200.200.133:5432/credit_tracker}
+ct.signal-data-source-settings.url=${DB_URL_SIGNAL:jdbc:postgresql://10.200.200.133:5432/credit_tracker?currentSchema=test_2}
 ct.signal-data-source-settings.username=${USERNAME_SIGNAL:credit_tracker_user}
 ct.signal-data-source-settings.password=${PASSWORD_SIGNAL:-}
-ct.signal-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SIGNAL:10}
+ct.signal-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SIGNAL:40}
 ct.signal-data-source-settings.minimum-idle=${MINIMUM_IDLE_SIGNAL:10}
 ct.signal-data-source-settings.connection-timeout=${CONNECTION_TIMEOUT_SIGNAL:10000}
 ct.signal-data-source-settings.validation-timeout=${VALIDATION_TIMEOUT_SIGNAL:5000}
@@ -39,7 +39,7 @@ ct.scoring-data-source-settings.driver=com.ibm.db2.jcc.DB2Driver
 ct.scoring-data-source-settings.url=${DB_URL_SCORING:jdbc:db2://10.230.227.101:2668/cprosd22:currentSchema=TEST_2_INDIC;}
 ct.scoring-data-source-settings.username=${USERNAME_SCORING:tester}
 ct.scoring-data-source-settings.password=${PASSWORD_SCORING:tester}
-ct.scoring-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SCORING:10}
+ct.scoring-data-source-settings.maximum-pool-size=${MAXIMUM_POOL_SIZE_SCORING:30}
 ct.scoring-data-source-settings.minimum-idle=${MINIMUM_IDLE_SCORING:10}
 ct.scoring-data-source-settings.connection-timeout=${CONNECTION_TIMEOUT_SCORING:10000}
 ct.scoring-data-source-settings.validation-timeout=${VALIDATION_TIMEOUT_SCORING:5000}
@@ -56,7 +56,7 @@ spring.servlet.multipart.max-request-size=${ACCEPTED_ARCHIVE_SIZE:2048MB}
 ct.xsd-validation.monitoring-request-path=monitoring-request.xsd
 ct.xsd-validation.signal-package-path=signal-package.xsd
 
-ct.signals-root-uri=${SIGNALS_ROOT_URI:http://10.230.227.102:11094/signalRUalfa/}
+ct.signals-root-uri=${SIGNALS_ROOT_URI:http://10.230.227.25:9888/signalRUalfa/}
 ct.signals-token=${SIGNALS_TOKEN:gttcHxwcGojoAvZK}
 ct.signals-user=${SIGNALS_USER:0001ZZ000001}
 
@@ -65,8 +65,53 @@ ct.load-dir=/opt/app/reports
 
 # min
 ct.monitoring-interval=${MONITORING_INTERVAL:15}
+ct.monitoring-membercodes=${MONITORING_MEMBERCODES:0001XX,0001BB}
 # days
-ct.update-flags-interval=${UPDATE_FLAGS_INTERVAL:1}
+ct.update-flags-cron-expression=${UPDATE_FLAGS_CRON_EXPRESSION:0 25 18 * * *}
+ct.new-flags-cron-expression=${NEW_FLAGS_CRON_EXPRESSION:0 25 18 * * *}
 ct.days-to-update-flags=${DAYS_TO_UPDATE_FLAGS:30}
+ct.update-fid-cron-expression=${UPDATE_FIDS_CRON_EXPRESSION:0 10 12 * * *}
+ct.days-to-update-fids=${DAYS_TO_UPDATE_FIDS:30}
+
+ct.db-version=${DB_VERSION:1234}
 
 ct.encoding=windows-1251
+
+
+#----------------------------Pipeline settings----------------------------#
+#batch size for saving in phone_fid_map
+ct.saver-subject-batch-size=${SAVER_SUBJECT_BATCH_SIZE:15000}
+#limit on select phone_fid_map
+ct.pipeline.subject-batch-size=${PIPELINE_SUBJECT_BATCH_SIZE:10000}
+
+#STAGES SETTINGS
+#select NAME + FID BY PHONE
+ct.pipeline.name-enricher-stage-concurrency=${PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY:20}
+ct.pipeline.name-enricher-queue-capacity=${PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY:20}
+#select ID BY FID
+ct.pipeline.id-enricher-stage-concurrency=${PIPELINE_ID_ENRICHER_STAGE_CONCURRENCY:10}
+ct.pipeline.id-enricher-queue-capacity=${PIPELINE_ID_ENRICHER_QUEUE_CAPACITY:20}
+#Save part of signal xml file and merge; only one thread
+ct.pipeline.merger-queue-capacity=${PIPELINE_MERGER_QUEUE_CAPACITY:50}
+
+#STAGES SETTINGS
+#select NAME + FID BY PHONE
+ct.pipeline.name-updater-stage-concurrency=${PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY:20}
+ct.pipeline.name-updater-queue-capacity=${PIPELINE_NAME_UPDATER_QUEUE_CAPACITY:20}
+#select ID BY FID
+ct.pipeline.id-updater-stage-concurrency=${PIPELINE_ID_UPDATER_STAGE_CONCURRENCY:10}
+ct.pipeline.id-updater-queue-capacity=${PIPELINE_ID_UPDATER_QUEUE_CAPACITY:20}
+
+
+#select and calc flags
+ct.flag-updater.concurrency=${FLAG_UPDATER_CONCURRENCY:10}
+ct.flag-updater.queue-capacity=${FLAG_UPDATER_QUEUE_CAPACITY:100}
+
+#Backoff settings
+ct.pipeline.error-attempt-count=${PIPELINE_ERROR_ATTEMPT_COUNT:5}
+ct.pipeline.error-attempt-delay=${PIPELINE_ERROR_ATTEMPT_DELAY:2S}
+ct.pipeline.retryable-exceptions=${PIPELINE_RETRYABLE_EXCEPTIONS:org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException}
+
+#Pipeline global settings
+ct.pipeline.global-timeout=${PIPELINE_GLOBAL_TIMEOUT:2H}
+ct.pipeline.parallel-requests=${PIPELINE_PARALLEL_REQUESTS:3}

docker_compose/credit_tracker_config.env

@@ -1,14 +1,16 @@
 #commons
-APP_VERSION=1.20
-REGISTRY_URL=rh8-vm3.nbki.msk:5000
-JMX_PORT=
+APP_VERSION=2.0
+REGISTRY_URL=10.230.238.56:5000
+JMX_MAIN_PORT=11001
+JMX_MONITORING_PORT=11002
 JMX_HOSTNAME=
 
 #volumes
-REPORT_DIR=D:\folder_name\signals
-LOGS_DIR=D:\folder_name\signals\logs\app
-JSON_LOGS_DIR=D:\folder_name\signals\logs\json
-ACCESS_LOGS_DIR=D:\folder_name\signals\logs\access
+REPORT_MAIN_DIR=/opt/credit_tracker_docker/main/reports
+REPORT_MONITOR_DIR=/opt/credit_tracker_docker/monitor/reports
+LOGS_DIR=/opt/credit_tracker_docker/logs
+JSON_LOGS_DIR=/opt/credit_tracker_docker/logs/json
+ACCESS_LOGS_DIR=/opt/credit_tracker_docker/logs/access
 
 #APM settings
 APM_SERVICE_NAME=credit-tracker
@@ -70,8 +72,31 @@ SIGNALS_USER=0001ZZ000001
 
 # min
 MONITORING_INTERVAL=15
+MONITORING_MEMBERCODES=0001XX,0001BB
 # days
-UPDATE_FLAGS_INTERVAL=1
 DAYS_TO_UPDATE_FLAGS=30
 
+UPDATE_FLAGS_CRON_EXPRESSION=0 25 18 * * *
+NEW_FLAGS_CRON_EXPRESSION=0 25 18 * * *
+UPDATE_FIDS_CRON_EXPRESSION=0 10 12 * * *
+DAYS_TO_UPDATE_FIDS=30
+
+SAVER_SUBJECT_BATCH_SIZE=15000
+PIPELINE_SUBJECT_BATCH_SIZE=10000
+PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=20
+PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=20
+PIPELINE_ID_ENRICHER_STAGE_CONCURRENCY=10
+PIPELINE_ID_ENRICHER_QUEUE_CAPACITY=20
+PIPELINE_MERGER_QUEUE_CAPACITY=50
+PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY=20
+PIPELINE_NAME_UPDATER_QUEUE_CAPACITY=20
+PIPELINE_ID_UPDATER_STAGE_CONCURRENCY=10
+PIPELINE_ID_UPDATER_QUEUE_CAPACITY=20
+FLAG_UPDATER_CONCURRENCY=10
+FLAG_UPDATER_QUEUE_CAPACITY=100
+PIPELINE_ERROR_ATTEMPT_COUNT=5
+PIPELINE_ERROR_ATTEMPT_DELAY=2S
+PIPELINE_RETRYABLE_EXCEPTIONS=org.springframework.dao.DataAccessException,org.springframework.dao.DataIntegrityViolationException}
+PIPELINE_GLOBAL_TIMEOUT=2H
+PIPELINE_PARALLEL_REQUESTS=3
 

docker_compose/docker-compose.yml

@@ -6,18 +6,23 @@ services:
     volumes:
       - ${LOGS_DIR}:/opt/app/logs/app
       - ${JSON_LOGS_DIR}:/opt/app/logs/json
-      - ${REPORT_DIR}:/opt/app/reports
+      - shared_data:/opt/app/reports
+      - ${REPORT_MAIN_DIR}:/opt/app/reports
       - ${ACCESS_LOGS_DIR}:/opt/app/logs/access
     ports:
       - "${HOST_PORT}:${SERVER_PORT}"
     environment:
+      - APP_JAR=main
+      - XMX=4G
+      - XMS=1G
       - TZ=Europe/Moscow
-      - APM_SERVICE_NAME=${APM_SERVICE_NAME}
+      - ${JVM_EXTRA_OPTS:-} \
+      - APM_SERVICE_NAME=${APM_SERVICE_NAME}-${APP_JAR}
       - APM_SERVER_URLS=${APM_SERVER_URLS}
       - APM_ENVIRONMENT=${APM_ENVIRONMENT}
       - APM_SERVICE_NODE_NAME=${APM_SERVICE_NODE_NAME}
       - APM_SERVICE_VERSION=${APP_VERSION}
-      - JMX_PORT=${JMX_PORT}
+      - JMX_PORT=${JMX_MAIN_PORT}
       - JMX_HOSTNAME=${JMX_HOSTNAME}
       - SERVER_PORT=${SERVER_PORT}
       - DB_URL_INDIC=${DB_URL_INDIC}
@@ -59,5 +64,90 @@ services:
       - SIGNALS_TOKEN=${SIGNALS_TOKEN}
       - SIGNALS_USER=${SIGNALS_USER}
       - MONITORING_INTERVAL=${MONITORING_INTERVAL}
-      - UPDATE_FLAGS_INTERVAL=${UPDATE_FLAGS_INTERVAL}
+      - MONITORING_MEMBERCODES=${MONITORING_MEMBERCODES}
+      - UPDATE_FLAGS_CRON_EXPRESSION=${UPDATE_FLAGS_CRON_EXPRESSION}
+      - NEW_FLAGS_CRON_EXPRESSION=${NEW_FLAGS_CRON_EXPRESSION}
+      - UPDATE_FIDS_CRON_EXPRESSION=${UPDATE_FIDS_CRON_EXPRESSION}
+      - DAYS_TO_UPDATE_FIDS=${DAYS_TO_UPDATE_FIDS}
       - DAYS_TO_UPDATE_FLAGS=${DAYS_TO_UPDATE_FLAGS}
+      - SAVER_SUBJECT_BATCH_SIZE=${SAVER_SUBJECT_BATCH_SIZE}
+      - PIPELINE_SUBJECT_BATCH_SIZE=${PIPELINE_SUBJECT_BATCH_SIZE}
+      - PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY=${PIPELINE_NAME_ENRICHER_STAGE_CONCURRENCY}
+      - PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY=${PIPELINE_NAME_ENRICHER_QUEUE_CAPACITY}
+      - PIPELINE_ID_ENRICHER_STAGE_CONCURRENCY=${PIPELINE_ID_ENRICHER_STAGE_CONCURRENCY}
+      - PIPELINE_ID_ENRICHER_QUEUE_CAPACITY=${PIPELINE_ID_ENRICHER_QUEUE_CAPACITY}
+      - PIPELINE_MERGER_QUEUE_CAPACITY=${PIPELINE_MERGER_QUEUE_CAPACITY}
+      - PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY=${PIPELINE_NAME_UPDATER_STAGE_CONCURRENCY}
+      - PIPELINE_NAME_UPDATER_QUEUE_CAPACITY=${PIPELINE_NAME_UPDATER_QUEUE_CAPACITY}
+      - PIPELINE_ID_UPDATER_STAGE_CONCURRENCY=${PIPELINE_ID_UPDATER_STAGE_CONCURRENCY}
+      - PIPELINE_ID_UPDATER_QUEUE_CAPACITY=${PIPELINE_ID_UPDATER_QUEUE_CAPACITY}
+      - FLAG_UPDATER_CONCURRENCY=${FLAG_UPDATER_CONCURRENCY}
+      - FLAG_UPDATER_QUEUE_CAPACITY=${FLAG_UPDATER_QUEUE_CAPACITY}
+      - PIPELINE_ERROR_ATTEMPT_COUNT=${PIPELINE_ERROR_ATTEMPT_COUNT}
+      - PIPELINE_ERROR_ATTEMPT_DELAY=${PIPELINE_ERROR_ATTEMPT_DELAY}
+      - PIPELINE_RETRYABLE_EXCEPTIONS=${PIPELINE_RETRYABLE_EXCEPTIONS}
+      - PIPELINE_GLOBAL_TIMEOUT=${PIPELINE_GLOBAL_TIMEOUT}
+      - PIPELINE_PARALLEL_REQUESTS=${PIPELINE_PARALLEL_REQUESTS}
+    command: ["/opt/app/start-credit-tracker.sh"]
+
+  credit-tracker-monitoring:
+    image: ${REGISTRY_URL}/credit-tracker:${APP_VERSION}
+    container_name: credit-tracker-monitoring
+    restart: always
+    volumes:
+      - shared_data:/opt/app/reports
+      - ${LOGS_DIR}:/opt/app/logs/app
+      - ${JSON_LOGS_DIR}:/opt/app/logs/json
+    environment:
+      - APP_JAR=monitoring
+      - TZ=Europe/Moscow
+      - XMX=4G
+      - XMS=1G
+      - APM_SERVICE_NAME=${APM_SERVICE_NAME}-${APP_JAR}
+      - APM_SERVER_URLS=${APM_SERVER_URLS}
+      - APM_ENVIRONMENT=${APM_ENVIRONMENT}
+      - APM_SERVICE_NODE_NAME=${APM_SERVICE_NODE_NAME}
+      - APM_SERVICE_VERSION=${APP_VERSION}
+      - JMX_PORT=${JMX_MONITORING_PORT}
+      - JMX_HOSTNAME=${JMX_HOSTNAME}
+      - DB_URL_INDIC=${DB_URL_INDIC}
+      - USERNAME_INDIC=${USERNAME_INDIC}
+      - PASSWORD_INDIC=${PASSWORD_INDIC}
+      - MAXIMUM_POOL_SIZE_INDIC=${MAXIMUM_POOL_SIZE_INDIC}
+      - MINIMUM_IDLE_INDIC=${MINIMUM_IDLE_INDIC}
+      - CONNECTION_TIMEOUT_INDIC=${CONNECTION_TIMEOUT_INDIC}
+      - VALIDATION_TIMEOUT_INDIC=${VALIDATION_TIMEOUT_INDIC}
+      - IDLE_TIMEOUT_INDIC=${IDLE_TIMEOUT_INDIC}
+      - MAX_LIFETIME_INDIC=${MAX_LIFETIME_INDIC}
+      - KEEPALIVE_TIME_INDIC=${KEEPALIVE_TIME_INDIC}
+      - LEAK_DETECTION_THRESHOLD_INDIC=${LEAK_DETECTION_THRESHOLD_INDIC}
+      - DB_URL_SIGNAL=${DB_URL_SIGNAL}
+      - USERNAME_SIGNAL=${USERNAME_SIGNAL}
+      - PASSWORD_SIGNAL=${PASSWORD_SIGNAL}
+      - MAXIMUM_POOL_SIZE_SIGNAL=${MAXIMUM_POOL_SIZE_SIGNAL}
+      - MINIMUM_IDLE_SIGNAL=${MINIMUM_IDLE_SIGNAL}
+      - CONNECTION_TIMEOUT_SIGNAL=${CONNECTION_TIMEOUT_SIGNAL}
+      - VALIDATION_TIMEOUT_SIGNAL=${VALIDATION_TIMEOUT_SIGNAL}
+      - IDLE_TIMEOUT_SIGNAL=${IDLE_TIMEOUT_SIGNAL}
+      - MAX_LIFETIME_SIGNAL=${MAX_LIFETIME_SIGNAL}
+      - KEEPALIVE_TIME_SIGNAL=${KEEPALIVE_TIME_SIGNAL}
+      - LEAK_DETECTION_THRESHOLD_SIGNAL=${LEAK_DETECTION_THRESHOLD_SIGNAL}
+      - DB_URL_SCORING=${DB_URL_SCORING}
+      - USERNAME_SCORING=${USERNAME_SCORING}
+      - PASSWORD_SCORING=${PASSWORD_SCORING}
+      - MAXIMUM_POOL_SIZE_SCORING=${MAXIMUM_POOL_SIZE_SCORING}
+      - MINIMUM_IDLE_SCORING=${MINIMUM_IDLE_SCORING}
+      - CONNECTION_TIMEOUT_SCORING=${CONNECTION_TIMEOUT_SCORING}
+      - VALIDATION_TIMEOUT_SCORING=${VALIDATION_TIMEOUT_SCORING}
+      - IDLE_TIMEOUT_SCORING=${IDLE_TIMEOUT_SCORING}
+      - MAX_LIFETIME_SCORING=${MAX_LIFETIME_SCORING}
+      - KEEPALIVE_TIME_SCORING=${KEEPALIVE_TIME_SCORING}
+      - LEAK_DETECTION_THRESHOLD_SCORING=${LEAK_DETECTION_THRESHOLD_SCORING}
+      - SIGNALS_ROOT_URI=${SIGNALS_ROOT_URI}
+      - SIGNALS_TOKEN=${SIGNALS_TOKEN}
+      - SIGNALS_USER=${SIGNALS_USER}
+      - MONITORING_INTERVAL=${MONITORING_INTERVAL}
+    command: ["/opt/app/start-credit-tracker.sh"]
+
+volumes:
+  shared_data:

pom.xml

@@ -10,16 +10,17 @@
     </parent>
     <groupId>ru.nbch</groupId>
     <artifactId>credit-tracker</artifactId>
-    <version>1.20</version>
+    <version>2.21.14</version>
     <name>credit_tracker</name>
     <description>Credit Tracker Application</description>
 
     <properties>
-        <java.version>17</java.version>
+        <java.version>21</java.version>
         <git.info.prefix>git_info</git.info.prefix>
         <git.info.filename>git.properties</git.info.filename>
-        <registry.url>10.230.238.56:5000</registry.url>
-        <!--        <registry.url>rh8-vm3.nbki.msk:5000</registry.url>-->
+        <registry.url>nexus-dev.nbki.msk</registry.url>
+        <jar.main.name>main</jar.main.name>
+        <jar.monitoring.name>monitoring</jar.monitoring.name>
     </properties>
 
     <dependencies>
@@ -41,6 +42,10 @@
             <artifactId>postgresql</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.ibm.db2</groupId>
             <artifactId>jcc</artifactId>
@@ -67,6 +72,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.codehaus.janino</groupId>
+            <artifactId>janino</artifactId>
+            <version>3.1.12</version>
+        </dependency>
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-core</artifactId>
@@ -155,8 +165,8 @@
                         <configuration>
                             <images>
                                 <image>
-                                    <name>${project.artifactId}:${project.version}</name>
-                                    <registry>${registry.url}</registry>
+                                    <name>nexus-dev.nbki.msk/${project.artifactId}:${project.version}</name>
+<!--                                    <registry>${registry.url}</registry>-->
                                     <build>
                                         <dockerFile>${project.basedir}/Dockerfile</dockerFile>
                                         <assembly>
@@ -164,7 +174,13 @@
                                                 <files>
                                                     <file>
                                                         <source>
-                                                            ${project.build.directory}/${project.build.finalName}.jar
+                                                            ${project.build.directory}/${project.build.finalName}-${jar.main.name}.jar
+                                                        </source>
+                                                        <outputDirectory>.</outputDirectory>
+                                                    </file>
+                                                    <file>
+                                                        <source>
+                                                            ${project.build.directory}/${project.build.finalName}-${jar.monitoring.name}.jar
                                                         </source>
                                                         <outputDirectory>.</outputDirectory>
                                                     </file>
@@ -229,14 +245,74 @@
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
-                <configuration>
-                    <excludes>
-                        <exclude>
-                            <groupId>org.projectlombok</groupId>
-                            <artifactId>lombok</artifactId>
-                        </exclude>
-                    </excludes>
-                </configuration>
+
+                <executions>
+                    <execution>
+                        <id>repackage</id>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <skip>true</skip>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>build-common</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <mainClass>ru.nbch.credit_tracker.service.mode.common.CTMainApplication</mainClass>
+                            <classifier>${jar.main.name}</classifier>
+                            <attach>true</attach>
+                            <excludes>
+                                <exclude>
+                                    <groupId>org.projectlombok</groupId>
+                                    <artifactId>lombok</artifactId>
+                                </exclude>
+                            </excludes>
+                        </configuration>
+                    </execution>
+
+                    <execution>
+                        <id>build-monitoring</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <mainClass>ru.nbch.credit_tracker.service.mode.monitoring.CTMonitoringApplication</mainClass>
+                            <classifier>${jar.monitoring.name}</classifier>
+                            <attach>true</attach>
+                            <excludes>
+                                <exclude>
+                                    <groupId>org.projectlombok</groupId>
+                                    <artifactId>lombok</artifactId>
+                                </exclude>
+                            </excludes>
+                        </configuration>
+                    </execution>
+
+                    <execution>
+                        <id>build-migration</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <mainClass>ru.nbch.credit_tracker.service.mode.migration.CTMigrationApplication</mainClass>
+                            <classifier>migration</classifier>
+                            <attach>true</attach>
+                            <excludes>
+                                <exclude>
+                                    <groupId>org.projectlombok</groupId>
+                                    <artifactId>lombok</artifactId>
+                                </exclude>
+                            </excludes>
+                        </configuration>
+                    </execution>
+                </executions>
             </plugin>
             <plugin>
                 <groupId>pl.project13.maven</groupId>

src/main/java/ru/nbch/credit_tracker/CreditTrackerApplication.java

@@ -1,23 +0,0 @@
-package ru.nbch.credit_tracker;
-
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.context.ConfigurableApplicationContext;
-import ru.nbch.credit_tracker.service.MonitoringService;
-import ru.nbch.credit_tracker.service.UpdateProcessingPackagesService;
-import ru.nbch.credit_tracker.service.UpdatingFlagsService;
-
-@SpringBootApplication
-public class CreditTrackerApplication {
-
-	public static void main(String[] args) {
-		ConfigurableApplicationContext ctx = SpringApplication.run(CreditTrackerApplication.class, args);
-		UpdateProcessingPackagesService updateProcessingPackagesService = ctx.getBean(UpdateProcessingPackagesService.class);
-		MonitoringService monitoringService = ctx.getBean(MonitoringService.class);
-		UpdatingFlagsService updatingFlagsService = ctx.getBean(UpdatingFlagsService.class);
-		updateProcessingPackagesService.run();
-		monitoringService.run();
-		updatingFlagsService.run();
-	}
-
-}

src/main/java/ru/nbch/credit_tracker/builder/PersonFactoryBuilderV2.java

@@ -1,16 +1,16 @@
 package ru.nbch.credit_tracker.builder;
 
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import ru.nbch.credit_tracker.entities.external_request.signals.Person;
-import ru.nbch.credit_tracker.service.indic.PersonService;
-import ru.nbch.credit_tracker.utils.CTUtil;
-
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import ru.nbch.credit_tracker.entities.external_request.signals.Person;
+import ru.nbch.credit_tracker.model.IdData;
+import ru.nbch.credit_tracker.model.NameData;
+import ru.nbch.credit_tracker.model.SubjectProfile;
 
 @Slf4j
-public class PersonFactoryBuilder {
+public class PersonFactoryBuilderV2 {
     private final static DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("dd.MM.yyyy");
 
     private final static String DEFAULT_DATE = "01.01.2000";
@@ -29,44 +29,38 @@ public class PersonFactoryBuilder {
     private final static String CONSENT_ISSUE_AUTHORITY = "Кем выдан";
 
 
-    private PersonService.PersonData personData;
-    private Long uid;
+    private SubjectProfile personData;
 
-    public static PersonFactoryBuilder builder() {
-        return new PersonFactoryBuilder();
+    public static PersonFactoryBuilderV2 builder() {
+        return new PersonFactoryBuilderV2();
     }
 
-    public PersonFactoryBuilder personData(PersonService.PersonData personData) {
+    public PersonFactoryBuilderV2 personData(SubjectProfile personData) {
         this.personData = personData;
         return this;
     }
 
-    public PersonFactoryBuilder uid(Long uid) {
-        this.uid = uid;
-        return this;
-    }
-
     public Person build() {
         if (personData == null) {
             return null;
         }
-        PersonService.NameData nameData = personData.getName();
-        PersonService.PassportData passportData = personData.getPassport();
+        NameData nameData = personData.getNameData();
+        IdData passportData = personData.getIdData();
         Person person = new Person();
         person.setS(nameData.getLastName());
         person.setN(nameData.getFirstName());
         person.setM(StringUtils.isNotBlank(nameData.getMiddleName()) ? nameData.getMiddleName() : null);
         person.setD(DATE_FORMAT.format(nameData.getBirthDate()));
 
-        person.setPn(getPn(passportData.getSerNum(), passportData.getIdNum()));
+        person.setPn(passportData.getSerNum());
         person.setPd(DATE_FORMAT.format(LocalDate.now())); // Do not use passport iss_date. Signal service could reject the package
-        person.setDt(getDt(passportData.getIdType(), passportData.getIdType2()));
+        person.setDt(passportData.getIdType());
         if (person.getDt() != null && person.getDt().equals("999")) {
             person.setOdt(ODT);
             person.setConsentOdt(ODT);
         }
         person.setIp(IP);
-        person.setUid(uid);
+        person.setUid(personData.getMapId());
         person.setOksm(OKSM);
         person.setConsentDate(DATE_FORMAT.format(LocalDate.now()));
         person.setConsentPurpose(CONSENT_PURPOSE);
@@ -90,35 +84,4 @@ public class PersonFactoryBuilder {
         person.setInqPurpose(INQ_PURPOSE);
         return person;
     }
-
-    private String getPn(String serNum, String idNum) {
-        String normSerNum = CTUtil.normalize(serNum);
-        String normIdNum = CTUtil.normalize(idNum);
-        return StringUtils.isNotBlank(normSerNum) ? normSerNum + normIdNum : normIdNum;
-    }
-
-    private String getDt(String idType, String idType2) {
-        if (StringUtils.isNotBlank(idType2)) {
-            return idType2;
-        } else {
-            return switch (idType) {
-                case "21" -> "21";
-                case "22" -> "22.1";
-                case "09" -> "22.2";
-                case "26" -> "23";
-                case "04" -> "24";
-                case "07" -> "25";
-                case "14", "03" -> "27";
-                case "10" -> "31";
-                case "15" -> "32";
-                case "13" -> "37";
-                case "06", "27" -> "28";
-                case "01", "02" -> "999";
-                default -> {
-                    log.warn("ID_TYPE2 is blank and no matching found for ID_TYPE {}", idType);
-                    yield "";
-                }
-            };
-        }
-    }
 }

src/main/java/ru/nbch/credit_tracker/component/MonitoringRequestValidator.java

@@ -0,0 +1,103 @@
+package ru.nbch.credit_tracker.component;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.web.multipart.MultipartFile;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
+import ru.nbch.credit_tracker.enums.SignalError;
+import ru.nbch.credit_tracker.enums.XmlErrorFormat;
+import ru.nbch.credit_tracker.exceptions.SignalException;
+import ru.nbch.credit_tracker.model.MonitoringValidationError;
+import ru.nbch.credit_tracker.model.ValidationResult;
+import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
+import ru.nbch.credit_tracker.service.ValidationService;
+import ru.nbch.credit_tracker.utils.ZipUtils;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.clientExcp;
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.serverExcp;
+
+@Component
+@Slf4j
+public class MonitoringRequestValidator {
+    private final CreditTrackerSettings config;
+    private final ValidationService validationService;
+    private final StaxXmlProcessor staxXmlProcessor;
+
+    public MonitoringRequestValidator(CreditTrackerSettings config,
+                                      ValidationService validationService,
+                                      StaxXmlProcessor staxXmlProcessor) {
+        this.config = config;
+        this.validationService = validationService;
+        this.staxXmlProcessor = staxXmlProcessor;
+    }
+
+    public void validateZipContent(MultipartFile file) {
+        log.trace("Validating zip content");
+        if (file.isEmpty() || !ZipUtils.isZipFile(file)) {
+            throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
+        }
+
+        // Валидация Архива на количетсво файлов, размер и расширение
+        try (ZipInputStream zipInputStream = new ZipInputStream(file.getInputStream())) {
+            Optional<ZipEntry> xmlFileEntry = ZipUtils.extractValidEntry(zipInputStream);
+            if (xmlFileEntry.isEmpty()) {
+                throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
+            }
+            long maxXmlFileSize = config.getMaxXmlFileSize() * 1024 * 1024;
+            if (xmlFileEntry.get().getSize() > maxXmlFileSize) {
+                throw clientExcp(SignalError.ERROR_009, XmlErrorFormat.MonitoringError);
+            }
+        } catch (IOException e) {
+            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
+        }
+        log.trace("Validated zip content ok!");
+    }
+
+    public ValidationResult readAndValidateMonitoringRequest(Path filePath) {
+        log.debug("Validating monitoring request...");
+        ValidationResult validationResult = new ValidationResult();
+        AtomicBoolean subjectWasMeet = new AtomicBoolean(false);
+        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(filePath), 8 * 1024 * 1024)) {
+            staxXmlProcessor.read(in, MonitoringRequest.class,
+                request -> {
+                    try {
+                        subjectWasMeet.compareAndSet(false, true);
+                        validationService.validateMonitoringRequest(request);
+                    } catch (SignalException e) {
+                        validationResult.addError(new MonitoringValidationError(-1, "", e));
+                    } finally {
+                        if (request.getPackageName() != null && validationResult.getPackageName() == null) {
+                            validationResult.setPackageName(request.getPackageName());
+                        }
+                        if (request.getUserId() != null && validationResult.getUserId() == null) {
+                            validationResult.setUserId(request.getUserId());
+                        }
+                        if (request.getPassword() != null && validationResult.getPassword() == null) {
+                            validationResult.setPassword(request.getPassword());
+                        }
+                        if (request.getSignalTypes() != null && !request.getSignalTypes().isEmpty()
+                            && validationResult.getSignalTypes().isEmpty()) {
+                            validationResult.setSignalTypes(request.getSignalTypes());
+                        }
+                        request.getSubjects().getSubject().clear();
+                    }
+                },
+                request -> subjectWasMeet.get() && request.getSubjects().getSubject().isEmpty(),
+                9010, "/MonitoringRequest/Subjects/Subject");
+        } catch (IOException e) {
+            throw new RuntimeException("Error while parsing and validating monitoring request", e);
+        }
+        log.debug("Validating monitoring request {}!", validationResult.getValidationErrors().isEmpty() ? "ok" : "fail");
+        return validationResult;
+    }
+}

src/main/java/ru/nbch/credit_tracker/component/RepositoryTimingAspect.java

@@ -0,0 +1,28 @@
+package ru.nbch.credit_tracker.component;
+
+import lombok.extern.slf4j.Slf4j;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.springframework.stereotype.Component;
+
+@Aspect
+@Slf4j
+@Component
+public class RepositoryTimingAspect {
+    @Around("within(@org.springframework.stereotype.Repository *)")
+    public Object timeMapper(ProceedingJoinPoint pjp) throws Throwable {
+        long start = System.nanoTime();
+        try {
+            return pjp.proceed();
+        } finally {
+            long tookMs = (System.nanoTime() - start) / 1_000_000;
+            String method = pjp.getSignature().toShortString();
+            if (tookMs > 1000) {
+                log.warn("Slow jdbc template call {} took {} ms", method, tookMs);
+            } else {
+                log.debug("Jdbc template call {} took {} ms", method, tookMs);
+            }
+        }
+    }
+}

src/main/java/ru/nbch/credit_tracker/component/SimpleMessageResolver.java

@@ -1,10 +1,12 @@
-package ru.nbch.credit_tracker.utils.error;
+package ru.nbch.credit_tracker.component;
 
+import org.springframework.stereotype.Component;
 import ru.nbch.credit_tracker.enums.EnumKeyMessage;
 
 import java.util.Arrays;
 
 
+@Component
 public class SimpleMessageResolver {
     public String resolve(EnumKeyMessage errorMessage) {
         if (errorMessage == null) return "null";

src/main/java/ru/nbch/credit_tracker/component/comparator/DeletePackageStatusComparator.java

@@ -0,0 +1,27 @@
+package ru.nbch.credit_tracker.component.comparator;
+
+import java.util.Comparator;
+import java.util.function.ToIntFunction;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.enums.Status;
+
+public class DeletePackageStatusComparator {
+
+    public static Comparator<UserPackage> INSTANCE = Comparator
+            .comparingInt((ToIntFunction<UserPackage>) userPackage -> statusToInt(userPackage.getStatus()))
+            .thenComparingLong(UserPackage::getId);
+
+    public static int statusToInt(String str) {
+        try {
+            Status status = Status.valueOf(str);
+            return switch (status) {
+                case ON_MONITORING, PROCESSING_REPEATED, NO_FID -> 1;
+                case DELETED -> 0;
+                default -> -1;
+            };
+        } catch (Throwable e) {
+            return -1;
+        }
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/component/comparator/ReportListPackageStatusComparator.java

@@ -0,0 +1,35 @@
+package ru.nbch.credit_tracker.component.comparator;
+
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.function.ToIntFunction;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.enums.Status;
+
+public class ReportListPackageStatusComparator {
+
+    public static final EnumSet<Status> reportedStatuses = EnumSet.of(
+        Status.ON_MONITORING, Status.PROCESSING_REPEATED
+    );
+
+    public static Comparator<UserPackage> INSTANCE = Comparator
+            .comparingInt((ToIntFunction<UserPackage>) userPackage -> statusToInt(userPackage.getStatus()))
+            .thenComparingLong(UserPackage::getId);
+
+    /**
+     * статусы не входящие в {@code reportedStatuses}
+     * кинут ошибку в <code>ru.nbch.credit_tracker.facade.SignalFacadeV3#getReportList</code>
+     */
+    public static int statusToInt(String str) {
+        try {
+            Status status = Status.valueOf(str);
+            return switch (status) {
+                case ON_MONITORING, PROCESSING_REPEATED -> 1;
+                default -> -1;
+            };
+        } catch (Throwable e) {
+            return -1;
+        }
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/config/AppConfig.java

@@ -0,0 +1,20 @@
+package ru.nbch.credit_tracker.config;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class AppConfig {
+
+    @Bean
+    public ObjectMapper entityToJsonMapper() {
+        return new ObjectMapper()
+            .registerModule(new JavaTimeModule())
+            .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
+            .setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/datasource/DatabaseConfig.java

@@ -4,6 +4,8 @@ import com.zaxxer.hikari.HikariDataSource;
 import org.springframework.boot.jdbc.DataSourceBuilder;
 import ru.nbch.credit_tracker.config.settings.DataSourceSettings;
 
+import javax.sql.DataSource;
+
 public class DatabaseConfig {
 
     protected HikariDataSource createDataSource(DataSourceSettings settings) {

src/main/java/ru/nbch/credit_tracker/config/datasource/IndicDataSourceConfig.java

@@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.io.Resource;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
 import ru.nbch.credit_tracker.config.settings.DataSourceSettings;
 
@@ -41,6 +42,7 @@ public class IndicDataSourceConfig extends DatabaseConfig {
         dataSource.setMaxLifetime(indicSetings.maxLifetime());
         dataSource.setKeepaliveTime(indicSetings.keepaliveTime());
         dataSource.setLeakDetectionThreshold(indicSetings.leakDetectionThreshold());
+        dataSource.setAutoCommit(false);
 
         try (Connection connection = dataSource.getConnection()) {
             connection.isValid(3);
@@ -60,5 +62,10 @@ public class IndicDataSourceConfig extends DatabaseConfig {
         sqlSessionFactoryBean.setTypeHandlersPackage("ru.nbch.credit_tracker.config.datasource.handlers");
         return sqlSessionFactoryBean.getObject();
     }
+
+    @Bean(name = "jdbcTemplateIndic")
+    public NamedParameterJdbcTemplate getNamedParameterJdbcTemplate(@Qualifier("indicDataSource") DataSource dataSource) {
+        return new NamedParameterJdbcTemplate(dataSource);
+    }
 }
 

src/main/java/ru/nbch/credit_tracker/config/datasource/handlers/LongTypeHandler.java

@@ -0,0 +1,41 @@
+package ru.nbch.credit_tracker.config.datasource.handlers;
+
+import java.sql.CallableStatement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.apache.ibatis.type.BaseTypeHandler;
+import org.apache.ibatis.type.JdbcType;
+import org.apache.ibatis.type.MappedTypes;
+
+@MappedTypes(Long.class)
+public class LongTypeHandler extends BaseTypeHandler<Long> {
+    @Override
+    public void setNonNullParameter(
+        PreparedStatement ps, int i, Long parameter, JdbcType jdbcType
+    ) throws SQLException {
+        ps.setString(i, parameter.toString());
+    }
+
+    @Override
+    public Long getNullableResult(ResultSet rs, String columnName) throws SQLException {
+        return parse(rs.getString(columnName));
+    }
+
+    @Override
+    public Long getNullableResult(ResultSet rs, int columnIndex) throws SQLException {
+        return parse(rs.getString(columnIndex));
+    }
+
+    @Override
+    public Long getNullableResult(CallableStatement cs, int columnIndex) throws SQLException {
+        return parse(cs.getString(columnIndex));
+    }
+
+    private Long parse(String value) {
+        if (value == null || value.isBlank()) {
+            return null;
+        }
+        return Long.valueOf(value);
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/datasource/handlers/UUIDTypeHandler.java

@@ -0,0 +1,53 @@
+package ru.nbch.credit_tracker.config.datasource.handlers;
+
+import java.sql.CallableStatement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.UUID;
+import org.apache.ibatis.type.JdbcType;
+import org.apache.ibatis.type.MappedTypes;
+import org.apache.ibatis.type.TypeHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@MappedTypes({UUID.class})
+public class UUIDTypeHandler implements TypeHandler<UUID> {
+    private static final Logger log = LoggerFactory.getLogger(UUIDTypeHandler.class);
+
+    @Override
+    public void setParameter(PreparedStatement ps, int i, UUID parameter, JdbcType jdbcType) throws SQLException {
+        if (parameter == null) {
+            ps.setObject(i, null, Types.OTHER);
+        } else {
+            ps.setObject(i, parameter.toString(), Types.OTHER);
+        }
+    }
+
+    @Override
+    public UUID getResult(ResultSet rs, String columnName) throws SQLException {
+        return toUUID(rs.getString(columnName));
+    }
+
+    @Override
+    public UUID getResult(ResultSet rs, int columnIndex) throws SQLException {
+        return toUUID(rs.getString(columnIndex));
+    }
+
+    @Override
+    public UUID getResult(CallableStatement cs, int columnIndex) throws SQLException {
+        return toUUID(cs.getString(columnIndex));
+    }
+
+    private static UUID toUUID(String val) {
+        if (val != null && !val.isEmpty() && !val.isBlank()) {
+            try {
+                return UUID.fromString(val);
+            } catch (IllegalArgumentException e) {
+                log.warn("Bad UUID found: {}", val);
+            }
+        }
+        return null;
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/mdc/ClearMdc.java

@@ -0,0 +1,14 @@
+package ru.nbch.credit_tracker.config.mdc;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import ru.nbch.credit_tracker.utils.LogUtil;
+
+// Annotation to only clear the MDC key at method exit
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface ClearMdc {
+    String key() default LogUtil.mdcKey;
+}

src/main/java/ru/nbch/credit_tracker/config/mdc/MdcAspect.java

@@ -1,18 +1,27 @@
 package ru.nbch.credit_tracker.config.mdc;
 
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.aspectj.lang.ProceedingJoinPoint;
 import org.aspectj.lang.annotation.Around;
 import org.aspectj.lang.annotation.Aspect;
-import org.slf4j.MDC;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
 import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
+import ru.nbch.credit_tracker.utils.LogUtil;
 
 @Slf4j
 @Aspect
 public class MdcAspect {
 
+    // For methods that only require clearing the MDC at the end
+    @Around("@annotation(clearMdc)")
+    public Object clearMdcAdvice(ProceedingJoinPoint pjp, ClearMdc clearMdc) throws Throwable {
+        try {
+            return pjp.proceed();
+        } finally {
+            LogUtil.removeMdcLogKey(clearMdc.key());
+        }
+    }
+
     // For methods that set an MDC value (from a method argument) and then clear it at the end
     @Around("@annotation(setAndClearMdc)")
     public Object setAndClearMdcAdvice(ProceedingJoinPoint pjp, SetAndClearMdc setAndClearMdc) throws Throwable {
@@ -20,30 +29,18 @@ public class MdcAspect {
         int index = setAndClearMdc.argIndex();
         Object obj = (args != null && args.length > index) ? args[index] : null;
         if (obj instanceof MonitoringRequest request) {
-            if (request.getPackageName() != null) {
-                putMdcValue(setAndClearMdc.key(), request.getPackageName());
-            }
+            LogUtil.addMdcLogKey(setAndClearMdc.key(), request.getPackageName());
         } else if (obj instanceof UserPackage userPackage) {
-            if (userPackage.getPackageName() != null) {
-                putMdcValue(setAndClearMdc.key(), userPackage.getPackageName());
-            }
+            LogUtil.addMdcLogKey(setAndClearMdc.key(), userPackage.getPackageName());
         } else if (obj instanceof String packageName) {
-            putMdcValue(setAndClearMdc.key(), packageName);
+            LogUtil.addMdcLogKey(setAndClearMdc.key(), packageName);
         } else {
             log.debug("no packageName found for for {}", pjp.toShortString());
         }
         try {
             return pjp.proceed();
         } finally {
-            MDC.remove(setAndClearMdc.key());
-        }
-    }
-
-    private void putMdcValue(String key, String value) {
-        try {
-            MDC.put(key, "[" + value + "] ");
-        } catch (Throwable e) {
-            log.error("{}", ExceptionUtils.getStackTrace(e));
+            LogUtil.removeMdcLogKey(setAndClearMdc.key());
         }
     }
 }

src/main/java/ru/nbch/credit_tracker/config/mdc/SetAndClearMdc.java

@@ -4,12 +4,13 @@ import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import ru.nbch.credit_tracker.utils.LogUtil;
 
 // Annotation to set an MDC key based on a method argument and clear it after method execution
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.METHOD)
 public @interface SetAndClearMdc {
-    String key() default "ru.nbch.credit_tracker.log.mdc.key";
+    String key() default LogUtil.mdcKey;
     // The index of the method argument to use for the MDC value (defaults to the first argument)
     int argIndex() default 0;
 }

src/main/java/ru/nbch/credit_tracker/config/rest/GlobalExceptionHandler.java

@@ -1,5 +1,6 @@
 package ru.nbch.credit_tracker.config.rest;
 
+import java.nio.charset.Charset;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
@@ -8,24 +9,32 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.RestControllerAdvice;
 import org.springframework.web.multipart.MaxUploadSizeExceededException;
+import org.springframework.web.multipart.MultipartException;
+import ru.nbch.credit_tracker.component.SimpleMessageResolver;
 import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.user_response.errors.*;
+import ru.nbch.credit_tracker.entities.user_response.errors.CommonError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringActivePackagesError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringAllPackagesError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringDownloadError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringPackageDeleteError;
+import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringReportListError;
 import ru.nbch.credit_tracker.enums.SignalError;
 import ru.nbch.credit_tracker.exceptions.SignalClientException;
 import ru.nbch.credit_tracker.exceptions.SignalException;
 import ru.nbch.credit_tracker.exceptions.SignalServerException;
-import ru.nbch.credit_tracker.utils.error.SimpleMessageResolver;
-
-import java.nio.charset.Charset;
+import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
 
 @Slf4j
 @RestControllerAdvice
 public class GlobalExceptionHandler {
 
-    private final SimpleMessageResolver messageResolver = new SimpleMessageResolver();
+    private final SimpleMessageResolver messageResolver;
     private final String encoding;
 
-    public GlobalExceptionHandler(CreditTrackerSettings settings) {
+    public GlobalExceptionHandler(SimpleMessageResolver messageResolver,
+                                  CreditTrackerSettings settings) {
+        this.messageResolver = messageResolver;
         this.encoding = settings.getEncoding();
     }
 
@@ -42,34 +51,47 @@ public class GlobalExceptionHandler {
         return new ResponseEntity<>(error, headers, HttpStatus.BAD_REQUEST);
     }
 
-    @ExceptionHandler(value = SignalException.class, produces = MediaType.APPLICATION_XML_VALUE)
-    public ResponseEntity<CommonError> handleClientException(SignalException ex) {
+    @ExceptionHandler(value = MultipartException.class, produces = MediaType.APPLICATION_XML_VALUE)
+    public ResponseEntity<CommonError> handleSpecialException(MultipartException ex) {
+        log.warn("Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception " +
+            "[Request processing failed: {}", ExceptionUtils.getStackTrace(ex));
         HttpHeaders headers = new HttpHeaders();
         headers.setContentType(new MediaType("application", "xml", Charset.forName(encoding)));
 
-        String errorText = messageResolver.resolve(ex.getErrorMessage());
-
-        CommonError commonError = getCommonError(ex);
-        commonError.setStatus("error");
-        commonError.setErrorCode(ex.getErrorMessage().getSubject().getCode());
-        commonError.setErrorText(errorText);
-
-        if (ex instanceof SignalClientException) {
-            String packageName = ((SignalClientException) ex).getPackageName();
-            if(packageName != null ) {
-                log.warn("Response error: {}. PackageName: {}", errorText, packageName);
-            } else {
-                log.warn("Response error: {}", errorText);
-            }
-            return new ResponseEntity<>(commonError, headers, HttpStatus.BAD_REQUEST);
-        } else {
-            Exception cause = ((SignalServerException) ex).getCause();
-            log.error(cause.getMessage(), cause);
-            return new ResponseEntity<>(commonError, headers, HttpStatus.INTERNAL_SERVER_ERROR);
-        }
+        MonitoringError error = new MonitoringError();
+        error.setStatus("error");
+        error.setErrorCode(SignalError.ERROR_028.getCode());
+        error.setErrorText(SignalError.ERROR_028.getKey());
+        return new ResponseEntity<>(error, headers, HttpStatus.BAD_REQUEST);
     }
 
-    private static CommonError getCommonError(SignalException ex) {
+    @ExceptionHandler(value = SignalClientException.class, produces = MediaType.APPLICATION_XML_VALUE)
+    public ResponseEntity<CommonError> handleClientException(SignalClientException ex) {
+        CommonError err = getCommonError(ex);
+        String packageName = ex.getPackageName();
+        if(packageName != null ) {
+            log.warn("Response error: {}. PackageName: {}", err.getErrorText(), packageName);
+        } else {
+            log.warn("Response error: {}", err.getErrorText());
+        }
+        return new ResponseEntity<>(err, xmlHeaders(), HttpStatus.BAD_REQUEST);
+    }
+
+    @ExceptionHandler(value = SignalServerException.class, produces = MediaType.APPLICATION_XML_VALUE)
+    public ResponseEntity<CommonError> handleServerException(SignalServerException ex) {
+        CommonError commonError = getCommonError(ex);
+        Exception cause = ex.getCause();
+        log.error(cause.getMessage(), cause);
+        return new ResponseEntity<>(commonError, xmlHeaders(), HttpStatus.INTERNAL_SERVER_ERROR);
+    }
+
+    private HttpHeaders xmlHeaders() {
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(new MediaType("application", "xml", Charset.forName(encoding)));
+        return headers;
+    }
+
+    private CommonError getCommonError(SignalException ex) {
         CommonError commonError;
         switch (ex.getErrorFormat()) {
             case MonitoringError -> commonError = new MonitoringError();
@@ -80,6 +102,10 @@ public class GlobalExceptionHandler {
             case MonitoringPackageDeleteError -> commonError = new MonitoringPackageDeleteError();
             default -> throw new IllegalStateException("Unsupported error format: " + ex.getErrorFormat());
         }
+        commonError.setStatus("error");
+        commonError.setErrorCode(ex.getErrorMessage().getSubject().getCode());
+        String errorText = messageResolver.resolve(ex.getErrorMessage());
+        commonError.setErrorText(errorText);
         return commonError;
     }
 }

src/main/java/ru/nbch/credit_tracker/config/settings/CreditTrackerSettings.java

@@ -1,5 +1,6 @@
 package ru.nbch.credit_tracker.config.settings;
 
+import java.util.List;
 import lombok.Getter;
 import lombok.Setter;
 import org.springframework.boot.context.properties.ConfigurationProperties;
@@ -21,6 +22,10 @@ public class CreditTrackerSettings {
     private DataSourceSettings scoringDataSourceSettings;
     @NestedConfigurationProperty
     private XsdValidation xsdValidation;
+    @NestedConfigurationProperty
+    private PipelineSettings pipeline;
+    @NestedConfigurationProperty
+    private FlagUpdaterSettings flagUpdater;
 
     private String signalsRootUri;
     private String signalsToken;
@@ -29,8 +34,10 @@ public class CreditTrackerSettings {
     private Long maxXmlFileSize;
     private String loadDir;
     private Long monitoringInterval;
-    private Long updateFlagsInterval;
-    private Long daysToUpdateFlags;
-
+    private List<String> monitoringMembercodes;
+    private String updateFlagsCronExpression;
+    private Integer daysToUpdateFlags;
+    private Long daysToUpdateFids;
     private String encoding;
+    private Integer saverSubjectBatchSize;
 }

src/main/java/ru/nbch/credit_tracker/config/settings/FlagUpdaterSettings.java

@@ -0,0 +1,9 @@
+package ru.nbch.credit_tracker.config.settings;
+
+import org.springframework.boot.context.properties.bind.DefaultValue;
+
+public record FlagUpdaterSettings(
+    @DefaultValue("10") Integer concurrency,
+    @DefaultValue("100") Integer queueCapacity
+) {
+}

src/main/java/ru/nbch/credit_tracker/config/settings/PipelineSettings.java

@@ -0,0 +1,27 @@
+package ru.nbch.credit_tracker.config.settings;
+
+import java.time.Duration;
+import java.util.List;
+import org.springframework.boot.context.properties.bind.DefaultValue;
+
+//X это количество потоков Save
+//тогда: Name должно быть ~3x, Fid должно быть ~2x чтобы они поспевали друг за другом?
+public record PipelineSettings(
+    @DefaultValue("5000") Integer subjectBatchSize,
+    @DefaultValue("15") Integer nameEnricherStageConcurrency,
+    @DefaultValue("20") Integer nameEnricherQueueCapacity,
+    @DefaultValue("10") Integer idEnricherStageConcurrency,
+    @DefaultValue("15") Integer idEnricherQueueCapacity,
+    @DefaultValue("15") Integer nameUpdaterStageConcurrency,
+    @DefaultValue("20") Integer nameUpdaterQueueCapacity,
+    @DefaultValue("10") Integer idUpdaterStageConcurrency,
+    @DefaultValue("15") Integer idUpdaterQueueCapacity,
+    @DefaultValue("50") Integer mergerQueueCapacity,
+    @DefaultValue("5") Integer errorAttemptCount,
+    @DefaultValue("2s") Duration errorAttemptDelay,
+    @DefaultValue("5m") Duration globalTimeout,
+    @DefaultValue("3") Integer parallelRequests,
+    @DefaultValue("org.springframework.dao.DataAccessException")
+    List<Class<? extends Throwable>> retryableExceptions
+) {
+}

src/main/java/ru/nbch/credit_tracker/config/task/ExecutorsConfig.java

@@ -0,0 +1,29 @@
+package ru.nbch.credit_tracker.config.task;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+
+@Configuration
+public class ExecutorsConfig {
+
+    @Bean(destroyMethod = "shutdown")
+    public ExecutorService pipelineExecutorService() {
+        return Executors.newFixedThreadPool(15, r -> {
+            Thread t = new Thread(r);
+            t.setName("pipe-starter-%d".formatted(t.threadId()));
+            return t;
+        });
+    }
+
+    @Bean
+    public ExecutorService saverExecutorService(CreditTrackerSettings creditTrackerSettings) {
+        return Executors.newFixedThreadPool(creditTrackerSettings.getPipeline().parallelRequests(), r -> {
+            Thread t = new Thread(r);
+            t.setName("subject-saver-%d".formatted(t.threadId()));
+            return t;
+        });
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/task/PipelineStarterConfig.java

@@ -0,0 +1,55 @@
+package ru.nbch.credit_tracker.config.task;
+
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.mapper.signal.PhonesMapMapper;
+import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
+import ru.nbch.credit_tracker.model.impl.EmptyTerminatePayload;
+import ru.nbch.credit_tracker.model.impl.PartFilePayload;
+import ru.nbch.credit_tracker.model.impl.SignalXmlFilePayload;
+import ru.nbch.credit_tracker.model.impl.SubjectPayload;
+import ru.nbch.credit_tracker.model.impl.VoidPayload;
+import ru.nbch.credit_tracker.service.signal.PackageRecordsService;
+import ru.nbch.credit_tracker.service.task.SetupParam;
+import ru.nbch.credit_tracker.service.task.impl.KickPipeLineDefaultImpl;
+import ru.nbch.credit_tracker.service.task.impl.KickPipeLineRepeatedRestoreImpl;
+import ru.nbch.credit_tracker.service.task.impl.KickRecalcPipeLine;
+import ru.nbch.credit_tracker.service.task.impl.NoOpTerminator;
+import ru.nbch.credit_tracker.service.task.impl.XmlMergeCoordinator;
+import ru.nbch.credit_tracker.service.task.setup.IPipeLineStarter;
+import ru.nbch.credit_tracker.service.task.setup.IPipeLineTerminator;
+
+@Configuration
+public class PipelineStarterConfig {
+    @Bean
+    public Function<SetupParam, IPipeLineStarter<SubjectPayload>> pipelineStarterFactory(CreditTrackerSettings creditTrackerSettings,
+                                                                                         UserPackageMapper userPackageMapper,
+                                                                                         PackageRecordsService packageRecordsService,
+                                                                                         PhonesMapMapper phonesMapMapper) {
+        return (setupParam) -> switch (setupParam.mode()) {
+            case DEFAULT, RESEND ->
+                new KickPipeLineDefaultImpl(setupParam, userPackageMapper, packageRecordsService, creditTrackerSettings);
+            case RESTORE ->
+                new KickPipeLineDefaultImpl(setupParam, userPackageMapper, packageRecordsService, creditTrackerSettings);
+            case REPEATED_RESTORE ->
+                new KickPipeLineRepeatedRestoreImpl(setupParam, userPackageMapper, packageRecordsService, creditTrackerSettings);
+            case MAKE_XML_BY_REPEATED ->
+                new KickPipeLineRepeatedRestoreImpl(setupParam, userPackageMapper, packageRecordsService, creditTrackerSettings);
+            case RECALC_PERSONAL_DATA ->
+                new KickRecalcPipeLine(setupParam, userPackageMapper, phonesMapMapper, packageRecordsService, creditTrackerSettings);
+        };
+    }
+
+    @Bean
+    public Supplier<IPipeLineTerminator<PartFilePayload, SignalXmlFilePayload>> pipelineTerminatorFactory(CreditTrackerSettings creditTrackerSettings) {
+        return () -> new XmlMergeCoordinator(creditTrackerSettings);
+    }
+
+    @Bean
+    public Supplier<IPipeLineTerminator<VoidPayload, EmptyTerminatePayload>> noOpTerminatorFactory() {
+        return NoOpTerminator::new;
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/task/StageConfig.java

@@ -0,0 +1,43 @@
+package ru.nbch.credit_tracker.config.task;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.model.impl.PartFilePayload;
+import ru.nbch.credit_tracker.model.impl.SubjectPayload;
+import ru.nbch.credit_tracker.model.impl.VoidPayload;
+
+@Configuration
+public class StageConfig {
+
+    @Bean("nameEnricherQueueFactory")
+    public StageQueueFactory<SubjectPayload> nameEnricherQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getPipeline().nameEnricherQueueCapacity());
+    }
+
+    @Bean("nameUpdaterQueueFactory")
+    public StageQueueFactory<SubjectPayload> nameUpdaterQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getPipeline().nameUpdaterQueueCapacity());
+    }
+
+    @Bean("idEnricherQueueFactory")
+    public StageQueueFactory<SubjectPayload> idEnricherQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getPipeline().nameEnricherQueueCapacity());
+    }
+
+    @Bean("idUpdaterQueueFactory")
+    public StageQueueFactory<SubjectPayload> idUpdaterQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getPipeline().nameUpdaterQueueCapacity());
+    }
+
+    @Bean("noOpQueueFactory")
+    public StageQueueFactory<VoidPayload> noOpQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getPipeline().mergerQueueCapacity());
+    }
+
+    @Bean("partFileQueueFactory")
+    public StageQueueFactory<PartFilePayload> partFileQueueFactory(CreditTrackerSettings creditTrackerSettings) {
+        return new StageQueueFactory<>(creditTrackerSettings.getPipeline().mergerQueueCapacity());
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/config/task/StageQueueFactory.java

@@ -0,0 +1,33 @@
+package ru.nbch.credit_tracker.config.task;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import org.springframework.beans.factory.FactoryBean;
+import ru.nbch.credit_tracker.model.StagePayload;
+
+public class StageQueueFactory<T extends StagePayload> implements FactoryBean<BlockingQueue<T>> {
+
+    private final int capacity;
+    public StageQueueFactory(int capacity) {
+        this.capacity = capacity;
+    }
+
+    public StageQueueFactory() {
+        this(10);
+    }
+
+    @Override
+    public BlockingQueue<T> getObject() throws Exception {
+        return new ArrayBlockingQueue<>(capacity);
+    }
+
+    @Override
+    public Class<?> getObjectType() {
+        return BlockingQueue.class;
+    }
+
+    @Override
+    public boolean isSingleton() {
+        return false;
+    }
+}

src/main/java/ru/nbch/credit_tracker/config/xml/XmlConfig.java

@@ -1,6 +1,10 @@
 package ru.nbch.credit_tracker.config.xml;
 
 import jakarta.xml.bind.JAXBException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
@@ -12,12 +16,15 @@ import ru.nbch.credit_tracker.entities.user_response.errors.MonitoringError;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringSetupResponse;
 import ru.nbch.credit_tracker.service.xml.jaxb.IXmlProcessor;
 import ru.nbch.credit_tracker.service.xml.jaxb.XmlProcessorImplAnyClasses;
-import ru.nbch.credit_tracker.service.xml.stax.*;
-
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Supplier;
+import ru.nbch.credit_tracker.service.xml.stax.MonitoringReportErrorsStaxtReader;
+import ru.nbch.credit_tracker.service.xml.stax.MonitoringRequestStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlReader;
+import ru.nbch.credit_tracker.service.xml.stax.OnlineDownloadReportPersonsStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.SgnlReportDeletedStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.SgnlReportStatusStaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.StaxReader;
+import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
+import ru.nbch.credit_tracker.service.xml.stax.XmlProcessorStaxImpl;
 
 @Configuration
 public class XmlConfig {
@@ -39,6 +46,7 @@ public class XmlConfig {
     public StaxXmlProcessor staxXmlProcessor() {
         List<Supplier<StaxReader<?>>> staxReader = new ArrayList<>();
         staxReader.add(SgnlReportStatusStaxReader::new);
+        staxReader.add(SgnlReportDeletedStaxReader::new);
         staxReader.add(MonitoringRequestStaxReader::new);
         staxReader.add(OnlineDownloadReportPersonsStaxReader::new);
         staxReader.add(MonitoringSignalResponseStaxXmlReader::new);

src/main/java/ru/nbch/credit_tracker/controller/SignalControllerV3.java

@@ -3,8 +3,14 @@ package ru.nbch.credit_tracker.controller;
 import org.springframework.core.io.Resource;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
+import ru.nbch.credit_tracker.config.mdc.ClearMdc;
 import ru.nbch.credit_tracker.entities.user_request.active_packages.MonitoringActivePackagesRequest;
 import ru.nbch.credit_tracker.entities.user_request.all_packages.MonitoringAllPackagesRequest;
 import ru.nbch.credit_tracker.entities.user_response.active_packages.MonitoringActivePackagesResponse;
@@ -12,47 +18,75 @@ import ru.nbch.credit_tracker.entities.user_response.all_packages.MonitoringAllP
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringPackageDeleteResponse;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringReportListResponse;
 import ru.nbch.credit_tracker.entities.user_response.success.MonitoringSetupResponse;
-import ru.nbch.credit_tracker.facade.SignalFacade;
+import ru.nbch.credit_tracker.enums.SignalError;
+import ru.nbch.credit_tracker.enums.XmlErrorFormat;
+import ru.nbch.credit_tracker.exceptions.SignalException;
+import ru.nbch.credit_tracker.facade.SignalFacadeV3;
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.serverExcp;
 
 @RestController
 @RequestMapping("/api/v1/signal")
-public class SignalController {
+public class SignalControllerV3 {
 
-    private final SignalFacade signalFacade;
+    private final SignalFacadeV3 signalFacadeV3;
 
-    public SignalController(SignalFacade signalFacade) {
-        this.signalFacade = signalFacade;
+    public SignalControllerV3(SignalFacadeV3 signalFacadeV3) {
+        this.signalFacadeV3 = signalFacadeV3;
     }
 
     @PostMapping(value = "/set", produces = MediaType.APPLICATION_XML_VALUE)
+    @ClearMdc
     public ResponseEntity<MonitoringSetupResponse> setupMonitoring(@RequestParam("file") MultipartFile file) {
-        return signalFacade.setupMonitoring(file);
+        signalFacadeV3.setupMonitoringRequest(file);
+        return ResponseEntity.ok().body(new MonitoringSetupResponse());
     }
 
     @PostMapping(value = "/active_packages", consumes = MediaType.APPLICATION_XML_VALUE, produces = MediaType.APPLICATION_XML_VALUE)
     public ResponseEntity<MonitoringActivePackagesResponse> findActivePackages(@RequestBody MonitoringActivePackagesRequest activePackagesRequest) {
-        return signalFacade.findActivePackages(activePackagesRequest);
+        try {
+            return signalFacadeV3.findActivePackages(activePackagesRequest);
+        } catch (Exception e) {
+            throwOrWrap(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError, e);
+            return null;
+        }
     }
 
     @PostMapping(value = "/all_packages", consumes = MediaType.APPLICATION_XML_VALUE, produces = MediaType.APPLICATION_XML_VALUE)
     public ResponseEntity<MonitoringAllPackagesResponse> findAllPackages(@RequestBody MonitoringAllPackagesRequest allPackagesRequest) {
-        return signalFacade.findAllPackages(allPackagesRequest);
+        try {
+            return signalFacadeV3.findAllPackages(allPackagesRequest);
+        } catch (Exception e) {
+            throwOrWrap(SignalError.ERROR_013, XmlErrorFormat.MonitoringAllPackagesError, e);
+            return null;
+        }
     }
 
+
     @PostMapping(value = "/delete", params = {"username", "packageName"}, produces = MediaType.APPLICATION_XML_VALUE)
     public ResponseEntity<MonitoringPackageDeleteResponse> deletePackage(@RequestParam("username") String username, @RequestParam("packageName") String packageName) {
-        return signalFacade.deletePackage(username, packageName);
+        try {
+            return signalFacadeV3.deletePackage(username, packageName);
+        } catch (Exception e) {
+            throwOrWrap(SignalError.ERROR_017, XmlErrorFormat.MonitoringPackageDeleteError, e);
+            return null;
+        }
     }
 
     @GetMapping(value = "/report_list", params = {"username", "packageName"}, produces = MediaType.APPLICATION_XML_VALUE)
     public ResponseEntity<MonitoringReportListResponse> getReportList(@RequestParam("username") String username, @RequestParam("packageName") String packageName) {
-        return signalFacade.getReportList(username, packageName);
+        return signalFacadeV3.getReportList(username, packageName);
     }
 
     @GetMapping(value = "/download", params = "file", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
     public ResponseEntity<Resource> downloadFile(@RequestParam("file") String file) {
-        return signalFacade.downloadFile(file);
+        return signalFacadeV3.downloadFile(file);
     }
 
+    private static void throwOrWrap(SignalError error, XmlErrorFormat specific, Exception ex) {
+        if (ex instanceof SignalException) {
+            throw (SignalException) ex;
+        }
+        throw serverExcp(error, specific, ex);
+    }
 }
 

src/main/java/ru/nbch/credit_tracker/entities/app/AppVersion.java

@@ -0,0 +1,10 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class AppVersion {
+    private String dbVersion;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/FlagDetail.java

@@ -1,16 +1,20 @@
-package ru.nbch.credit_tracker.entities.dto;
-
-import lombok.*;
+package ru.nbch.credit_tracker.entities.app;
 
 import java.time.LocalDateTime;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 
 @Getter
 @Setter
-@AllArgsConstructor
 @NoArgsConstructor
+@AllArgsConstructor
 @Builder
-public class PhoneFidMap {
+public class FlagDetail {
     private Long id;
+    private Long mapId;
     private Long packageId;
     private String subjectId;
     private String phone;
@@ -18,5 +22,4 @@ public class PhoneFidMap {
     private Integer flagDebt5000;
     private Integer flag90days;
     private LocalDateTime calculatedAt;
-    private String errorText;
 }

src/main/java/ru/nbch/credit_tracker/entities/app/Hit.java

@@ -1,14 +1,15 @@
-package ru.nbch.credit_tracker.entities.dto;
+package ru.nbch.credit_tracker.entities.app;
 
+import java.time.LocalDateTime;
 import lombok.Getter;
 import lombok.Setter;
 
-import java.time.LocalDateTime;
-
 @Getter
 @Setter
+//@Builder
 public class Hit {
     private Long id;
+    private Long phone;
     private Long packageId;
     private String subjectId;
     private Integer signalType;

src/main/java/ru/nbch/credit_tracker/entities/app/PackageRecordWithPhone.java

@@ -0,0 +1,28 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import java.time.LocalDate;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+public class PackageRecordWithPhone {
+
+    //PackageRecordsMap fields
+    private Long packageId;
+    private String subjectId;
+    private String phone;
+
+    //PhonesMap fields
+    private Integer fid;
+    private String name1;
+    private String first;
+    private String middle;
+    private LocalDate birthDt;
+    private String serNum;
+    private String idType;
+    private Integer flagDebt5000;
+    private Integer flag90days;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/PackagesRecordsMap.java

@@ -0,0 +1,20 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class PackagesRecordsMap {
+    private Long id;
+    private Long packageId;
+    private String subjectId;
+    private Long phone_l;
+    private String errorText;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/PhonesMap.java

@@ -0,0 +1,30 @@
+package ru.nbch.credit_tracker.entities.app;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class PhonesMap {
+    private Long id;
+    private Integer fid;
+    private Long phone;
+    private String name1;
+    private String first;
+    private String middle;
+    private LocalDate birthDt;
+    private String serNum;
+    private String idType;
+    private Integer flagDebt5000;
+    private Integer flag90days;
+    private LocalDateTime searchedAt;
+    private LocalDateTime calculatedAt;
+}

src/main/java/ru/nbch/credit_tracker/entities/app/UserPackage.java

@@ -1,21 +1,27 @@
-package ru.nbch.credit_tracker.entities.dto;
-
-import lombok.Builder;
-import lombok.Getter;
-import lombok.Setter;
+package ru.nbch.credit_tracker.entities.app;
 
 import java.time.LocalDateTime;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 
 @Getter
 @Setter
 @Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class UserPackage {
     private Long id;
     private String userId;
     private String packageName;
+    private String signalTypes;
     private LocalDateTime createdAt;
     private String status;
     private LocalDateTime deactivatedAt;
     private String lastDownloadedReport;
     private LocalDateTime monitoringAt;
+    private String rejectStatus;
+    private String externalPackageName;
 }

src/main/java/ru/nbch/credit_tracker/entities/external_response/list_package/SgnlReportDeleted.java

@@ -0,0 +1,18 @@
+package ru.nbch.credit_tracker.entities.external_response.list_package;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Setter
+@Getter
+public class SgnlReportDeleted {
+    private Integer deleteFlag;
+
+    public Integer getDeleteFlag() {
+        return deleteFlag;
+    }
+
+    public void setDeleteFlag(Integer deleteFlag) {
+        this.deleteFlag = deleteFlag;
+    }
+}

src/main/java/ru/nbch/credit_tracker/entities/indic/Id.java

@@ -0,0 +1,24 @@
+package ru.nbch.credit_tracker.entities.indic;
+
+import java.time.LocalDate;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Id {
+    private Integer fid;
+    private String number;
+    private String serNum;
+    private String idNum;
+    private LocalDate issDate;
+    private String idType;
+    private String idType2;
+    private LocalDate fileSinceDate;
+}

src/main/java/ru/nbch/credit_tracker/entities/indic/Name.java

@@ -0,0 +1,25 @@
+package ru.nbch.credit_tracker.entities.indic;
+
+import java.time.LocalDate;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Name {
+    private Integer fid;
+    private String number;
+    private LocalDate fileSinceDate;
+    private LocalDate fileSinceDatePn;
+    private String first;
+    private String last;
+    private String middle;
+    private LocalDate birthDate;
+
+}

src/main/java/ru/nbch/credit_tracker/entities/indic/PaymentsFlag.java

@@ -0,0 +1,11 @@
+package ru.nbch.credit_tracker.entities.indic;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class PaymentsFlag {
+    private Integer fid;
+    private Integer flag;
+}

src/main/java/ru/nbch/credit_tracker/entities/user_request/monitoring_request/MonitoringRequest.java

@@ -1,7 +1,13 @@
 
 package ru.nbch.credit_tracker.entities.user_request.monitoring_request;
 
-import jakarta.xml.bind.annotation.*;
+import jakarta.xml.bind.annotation.XmlAccessType;
+import jakarta.xml.bind.annotation.XmlAccessorType;
+import jakarta.xml.bind.annotation.XmlElement;
+import jakarta.xml.bind.annotation.XmlRootElement;
+import jakarta.xml.bind.annotation.XmlType;
+import java.util.ArrayList;
+import java.util.List;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -26,6 +32,7 @@ public class MonitoringRequest {
     protected String packageName;
     @XmlElement(name = "Subjects", required = true)
     protected Subjects subjects;
+    protected List<Integer> signalTypes = null;
 
     public Subjects getSubjects() {
         if (subjects == null) {
@@ -40,6 +47,10 @@ public class MonitoringRequest {
         currentSubject = new Subject();
     }
 
+    public void createSignalTypes() {
+        signalTypes = new ArrayList<>();
+    }
+
     public Subject getCurrentSubject() {
         return currentSubject;
     }

src/main/java/ru/nbch/credit_tracker/entities/user_response/errors/CommonError.java

@@ -4,4 +4,5 @@ public interface CommonError {
     void setStatus(String status);
     void setErrorCode(String errorCode);
     void setErrorText(String errorText);
+    String getErrorText();
 }

src/main/java/ru/nbch/credit_tracker/enums/PoisonType.java

@@ -0,0 +1,7 @@
+package ru.nbch.credit_tracker.enums;
+
+public enum PoisonType {
+    NONE,
+    NORMALLY,
+    EXCEPTIONALLY
+}

src/main/java/ru/nbch/credit_tracker/enums/RejectedStatus.java

@@ -0,0 +1,17 @@
+package ru.nbch.credit_tracker.enums;
+
+public enum RejectedStatus {
+    OK("0"),
+    ERROR("1"),;
+
+
+    private final String key;
+
+    RejectedStatus(String key) {
+        this.key = key;
+    }
+
+    public String getKey() {
+        return key;
+    }
+}

src/main/java/ru/nbch/credit_tracker/enums/SignalError.java

@@ -21,7 +21,14 @@ public enum SignalError implements IEnumKeyCode {
     ERROR_019("019", "У данного пользователя не найдено ни одного пакета"),
     ERROR_020("020", "Пакет {} был отклонён при установке на мониторинг. Отчеты не формируются."),
     ERROR_021("021", "Обработка пакета {} завершилась с ошибкой, отчёты недоступны. Обратитесь в техническую поддержку."),
-    ERROR_022("022", "Пакет {} снят с мониторинга, отчёты больше недоступны.");
+    ERROR_022("022", "Пакет {} снят с мониторинга, отчёты больше недоступны."),
+    ERROR_023("023", "Пакет {} находится в обработке, удаление невозможно. Дождитесь его установки на мониторинг и повторите попытку."),
+    ERROR_024("024", "Пакет {} отклонен или при его установке произошла ошибка. Удаление невозможно."),
+    ERROR_025("025", "Пакет {} уже был удален ранее, удаление невозможно."),
+    ERROR_026("026", "Пакет {} обновляется. Удаление невозможно. Дождитесь завершения обновления и повторите попытку."),
+    ERROR_027("027", "Недопустимое значение в элементе SignalTypes"),
+    ERROR_028("028", "Некорректный формат запроса: в теле ожидается ZIP-архив"),
+    ;
 
 
     private final String code;

src/main/java/ru/nbch/credit_tracker/enums/Status.java

@@ -2,8 +2,11 @@ package ru.nbch.credit_tracker.enums;
 
 public enum Status {
     ON_MONITORING, // пакет стоит на мониторинге в оф. сервисе
+    NO_FID,
     REJECTED, // отклонен на стороне НБКИ
     PROCESSING, // идет поиск телефона и расчет флагов по пакету
+    PROCESSING_REPEATED,
+    ACCEPTED,
     DELETED, // пакет удален
     ERROR; // произошла ошибка при постановке пакета на мониторинг в оф. сервис
 }

src/main/java/ru/nbch/credit_tracker/facade/SignalFacade.java

@@ -1,436 +0,0 @@
-package ru.nbch.credit_tracker.facade;
-
-import lombok.extern.slf4j.Slf4j;
-import org.slf4j.MDC;
-import org.springframework.aop.framework.AopContext;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.EnableAspectJAutoProxy;
-import org.springframework.core.io.FileUrlResource;
-import org.springframework.core.io.Resource;
-import org.springframework.http.ResponseEntity;
-import org.springframework.stereotype.Service;
-import org.springframework.web.multipart.MultipartFile;
-import ru.nbch.credit_tracker.config.mdc.SetAndClearMdc;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.entities.user_request.active_packages.MonitoringActivePackagesRequest;
-import ru.nbch.credit_tracker.entities.user_request.all_packages.MonitoringAllPackagesRequest;
-import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
-import ru.nbch.credit_tracker.entities.user_response.active_packages.MonitoringActivePackagesResponse;
-import ru.nbch.credit_tracker.entities.user_response.active_packages.Package;
-import ru.nbch.credit_tracker.entities.user_response.active_packages.Packages;
-import ru.nbch.credit_tracker.entities.user_response.all_packages.MonitoringAllPackagesResponse;
-import ru.nbch.credit_tracker.entities.user_response.all_packages.PackageAll;
-import ru.nbch.credit_tracker.entities.user_response.all_packages.PackagesAll;
-import ru.nbch.credit_tracker.entities.user_response.success.MonitoringPackageDeleteResponse;
-import ru.nbch.credit_tracker.entities.user_response.success.MonitoringReportListResponse;
-import ru.nbch.credit_tracker.entities.user_response.success.MonitoringSetupResponse;
-import ru.nbch.credit_tracker.enums.SignalError;
-import ru.nbch.credit_tracker.enums.Status;
-import ru.nbch.credit_tracker.enums.XmlErrorFormat;
-import ru.nbch.credit_tracker.exceptions.SignalClientException;
-import ru.nbch.credit_tracker.service.MonitoringErrorProcesingService;
-import ru.nbch.credit_tracker.service.SignalService;
-import ru.nbch.credit_tracker.service.scoring.AuthorizationService;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
-import ru.nbch.credit_tracker.utils.CTUtil;
-import ru.nbch.credit_tracker.utils.ValidationService;
-import ru.nbch.credit_tracker.utils.ZipUtils;
-
-import javax.xml.stream.XMLStreamException;
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.LocalDateTime;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.clientExcp;
-import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.serverExcp;
-
-@Slf4j
-@Service
-@EnableAspectJAutoProxy(exposeProxy = true)
-public class SignalFacade {
-
-    private SignalFacade self;
-
-    private final CreditTrackerSettings config;
-    private final AuthorizationService authorizationService;
-    private final SignalService signalService;
-    private final SignalRestService signalRestService;
-    private final MonitoringErrorProcesingService monitoringErrorProcesingService;
-    private final ValidationService validationService;
-    private final StaxXmlProcessor staxXmlProcessor;
-
-    private final ExecutorService executor;
-    private final long maxXmlFileSize;
-
-
-    public SignalFacade(CreditTrackerSettings config,
-                        AuthorizationService authorizationService,
-                        SignalService signalService,
-                        SignalRestService signalRestService,
-                        MonitoringErrorProcesingService monitoringErrorProcesingService,
-                        ValidationService validationService,
-                        StaxXmlProcessor staxXmlProcessor,
-                        @Qualifier("executorService") ExecutorService executor) {
-        this.config = config;
-        this.authorizationService = authorizationService;
-        this.signalService = signalService;
-        this.signalRestService = signalRestService;
-        this.monitoringErrorProcesingService = monitoringErrorProcesingService;
-        this.validationService = validationService;
-        this.staxXmlProcessor = staxXmlProcessor;
-        this.executor = executor;
-        this.maxXmlFileSize = config.getMaxXmlFileSize() * 1024 * 1024;
-    }
-
-    public ResponseEntity<MonitoringSetupResponse> setupMonitoring(MultipartFile file) {
-        // первичные валидации
-        if (file.isEmpty() || !isZipFile(file)) {
-            throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
-        }
-
-        // Валидация Архива на количетсво файлов, размер и расширение
-        MonitoringRequest monitoringRequest;
-        Long packageId;
-        ZipInputStream zipInputStream = null;
-        String filePath;
-        try {
-            zipInputStream = new ZipInputStream(file.getInputStream());
-            Optional<ZipEntry> xmlFileEntry = ZipUtils.extractValidEntry(zipInputStream);
-            if (xmlFileEntry.isEmpty()) {
-                throw clientExcp(SignalError.ERROR_001, XmlErrorFormat.MonitoringError);
-            }
-            if (xmlFileEntry.get().getSize() > maxXmlFileSize) {
-                throw clientExcp(SignalError.ERROR_009, XmlErrorFormat.MonitoringError);
-            }
-
-            zipInputStream.close();
-            zipInputStream = new ZipInputStream(file.getInputStream());
-            zipInputStream.getNextEntry();
-
-            filePath = writeTmpFile(zipInputStream);
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
-        } finally {
-            if (zipInputStream != null) {
-                try {
-                    zipInputStream.close();
-                } catch (IOException e) {
-                    throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
-                }
-            }
-        }
-
-
-        // валидация xml
-        try {
-            monitoringRequest = readAndValidateMonitoringRequest(filePath);
-        } catch (Exception e) {
-            CTUtil.removeFile(filePath);
-            if (e.getCause() instanceof XMLStreamException) {
-                throw clientExcp(SignalError.ERROR_011, XmlErrorFormat.MonitoringError);
-            }
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
-        }
-
-        try {
-            // проверка авторизации
-            AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(monitoringRequest.getUserId(), monitoringRequest.getPassword());
-            if (!authResult.isSuccess()) {
-                signalService.rejectPackage(monitoringRequest);
-                throw clientExcp(authResult.getError(), monitoringRequest.getPackageName(), XmlErrorFormat.MonitoringError);
-            }
-
-            // проверка на уникальность пакета
-            if (signalService.isPackageNotUnique(monitoringRequest)) {
-                signalService.rejectPackage(monitoringRequest);
-                throw clientExcp(SignalError.ERROR_007, monitoringRequest.getPackageName(), XmlErrorFormat.MonitoringError);
-            }
-
-            packageId = signalService.registerPackage(monitoringRequest);
-            log.info("Package registered. PackageId {}. Status: {}", packageId, Status.PROCESSING.name());
-        } catch (Exception e) {
-            CTUtil.removeFile(filePath);
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
-        }
-
-        // в отдельном потоке запускаем регистрацию субъектов, поиск фидов и расчет флагов просрочек
-        getCurrentProxy().registerFidsAndDebts(packageId, monitoringRequest.getPackageName(), filePath);
-
-        return ResponseEntity.ok().body(new MonitoringSetupResponse());
-    }
-
-    private MonitoringRequest readAndValidateMonitoringRequest(String filePath) {
-        MonitoringRequest monitoringRequest;
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(filePath)), 8 * 1024 * 1024)) {
-            monitoringRequest = staxXmlProcessor.read(in, MonitoringRequest.class,
-                    request -> {
-                        try {
-                            validationService.validateMonitoringRequest(request);
-                        } catch (SignalClientException e) {
-                            if (request != null) {
-                                signalService.rejectPackage(request);
-                            }
-                            request = null;
-                            throw e;
-                        }
-                        request.getSubjects().getSubject().clear();
-                    },
-                    request -> request.getSubjects().getSubject().isEmpty()
-                    ,
-                    40000, "/MonitoringRequest/Subjects/Subject");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while parsing and validating monitoring request", e);
-        }
-        return monitoringRequest;
-    }
-
-    private String writeTmpFile(ZipInputStream zipInputStream) throws IOException {
-        String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator;
-        String filePath = folderPath + UUID.randomUUID() + "monitoring_request.xml";
-        CTUtil.createFolder(folderPath);
-
-        try (OutputStream out = Files.newOutputStream(Path.of(filePath))) {
-
-            byte[] buffer = new byte[8 * 1024 * 1024];
-            int bytesRead;
-            while ((bytesRead = zipInputStream.read(buffer)) > 0) {
-                out.write(buffer, 0, bytesRead);
-            }
-
-            return filePath;
-        }
-    }
-
-    @SetAndClearMdc(argIndex = 1)
-    public void registerFidsAndDebts(Long packageId, String packageName, String filePath) {
-        Map<String, String> context = MDC.getCopyOfContextMap();
-        executor.execute(() -> {
-            if (context != null) {
-                MDC.setContextMap(context);
-            }
-            try {
-                signalService.registerSubjects(filePath, packageId);
-                CTUtil.removeFile(filePath);
-                SignalService.UpdateResult updateResult = signalService.updateSubjects(packageId, null);
-                if (updateResult.isNoFidsFound()) {
-                    log.info("No fids found for PackageId: {} PackageName: {}. Package status ON_MONITORING. No registration in signal service.", packageId, packageName);
-                    signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-                    return;
-                }
-                if (signalRestService.setupMonitoring(packageId)) {
-                    LocalDateTime monitoringAt = LocalDateTime.now();
-                    log.info("Package was successfully registered in signal service. PackageId: {} PackageName: {}, MonitoringAt: {}", packageId, packageName, monitoringAt);
-                    signalService.updatePackageOnMonitoringSuccess(packageId, monitoringAt);
-                    monitoringErrorProcesingService.findErrors(packageId);
-                } else {
-                    log.info("Package registration was rejected by signal service. PackageId: {} PackageName: {}", packageId, packageName);
-                    signalService.updatePackageStatus(packageId, Status.ERROR);
-                }
-            } catch (Exception e) {
-                CTUtil.removeFile(filePath);
-                signalService.updatePackageStatus(packageId, Status.ERROR);
-                log.error("Ошибка при поиске фидов или расчете флагов просрочек. Package_id {}", packageId, e);
-            }
-        });
-    }
-
-    private boolean isZipFile(MultipartFile file) {
-        return file.getContentType() != null &&
-                (file.getContentType().equals("application/zip") || file.getContentType().equals("application/x-zip-compressed")) ||
-                file.getOriginalFilename() != null && file.getOriginalFilename().toLowerCase().endsWith(".zip");
-    }
-
-    public ResponseEntity<MonitoringActivePackagesResponse> findActivePackages(MonitoringActivePackagesRequest activePackagesRequest) {
-        try {
-            AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(activePackagesRequest.getUserId(), activePackagesRequest.getPassword());
-            if (!authResult.isSuccess()) {
-                throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringActivePackagesError);
-            }
-            List<UserPackage> activePackages = signalService.findActivePackages(activePackagesRequest.getUserId());
-
-            if (!activePackages.isEmpty()) {
-                MonitoringActivePackagesResponse response = new MonitoringActivePackagesResponse();
-                response.setStatus("OK");
-                response.setMembercode(activePackagesRequest.getUserId().substring(0, 6));
-                Packages packages = new Packages();
-                activePackages.forEach(data -> {
-                    Package pack = new Package();
-                    pack.setPackageName(data.getPackageName());
-                    pack.setCreatedAt(data.getCreatedAt() != null ? data.getCreatedAt().toString() : null);
-                    packages.getPackages().add(pack);
-                });
-                response.setPackages(packages);
-                return ResponseEntity.ok(response);
-            } else {
-                throw clientExcp(SignalError.ERROR_015, XmlErrorFormat.MonitoringActivePackagesError);
-            }
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringActivePackagesError, e);
-        }
-    }
-
-    public ResponseEntity<MonitoringAllPackagesResponse> findAllPackages(MonitoringAllPackagesRequest allPackagesRequest) {
-        try {
-            AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(allPackagesRequest.getUserId(), allPackagesRequest.getPassword());
-            if (!authResult.isSuccess()) {
-                throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringAllPackagesError);
-            }
-            List<UserPackage> activePackages = signalService.findAllPackages(allPackagesRequest.getUserId());
-
-            if (!activePackages.isEmpty()) {
-                MonitoringAllPackagesResponse response = new MonitoringAllPackagesResponse();
-                response.setStatus("OK");
-                response.setMembercode(allPackagesRequest.getUserId().substring(0, 6));
-                PackagesAll packages = new PackagesAll();
-                activePackages.forEach(data -> {
-                    PackageAll pack = new PackageAll();
-                    pack.setPackageName(data.getPackageName());
-                    pack.setCreatedAt(data.getCreatedAt() != null ? data.getCreatedAt().toString() : null);
-                    pack.setStatus(data.getStatus());
-                    if (data.getStatus().equals(Status.DELETED.name())) {
-                        pack.setDeactivatedAt(data.getDeactivatedAt() != null ? data.getDeactivatedAt().toString() : null);
-                    }
-                    packages.getPackages().add(pack);
-                });
-                response.setPackages(packages);
-                return ResponseEntity.ok(response);
-            } else {
-                throw clientExcp(SignalError.ERROR_019, XmlErrorFormat.MonitoringAllPackagesError);
-            }
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringAllPackagesError, e);
-        }
-    }
-
-    public ResponseEntity<MonitoringPackageDeleteResponse> deletePackage(String username, String packageName) {
-        try {
-            if (!signalService.isPackageExists(username, packageName)) {
-                throw clientExcp(SignalError.ERROR_016, XmlErrorFormat.MonitoringPackageDeleteError, packageName);
-            }
-
-            LocalDateTime deactivatedAt = LocalDateTime.now();
-            signalService.deletePackage(username, packageName, deactivatedAt);
-
-            MonitoringPackageDeleteResponse response = new MonitoringPackageDeleteResponse();
-            response.setStatus("Deleted");
-            response.setPackageName(packageName);
-            response.setDeactivatedAt(deactivatedAt.toString());
-
-            signalRestService.deletePackage(username, packageName);
-            log.info("Package {} deleted", packageName);
-            return ResponseEntity.ok(response);
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_017, XmlErrorFormat.MonitoringPackageDeleteError, e);
-        }
-    }
-
-    public ResponseEntity<MonitoringReportListResponse> getReportList(String username, String packageName) {
-        try {
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.PROCESSING)) {
-                throw clientExcp(SignalError.ERROR_018, XmlErrorFormat.MonitoringReportListError, packageName);
-            }
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.REJECTED)) {
-                throw clientExcp(SignalError.ERROR_020, XmlErrorFormat.MonitoringReportListError, packageName);
-            }
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.ERROR)) {
-                throw clientExcp(SignalError.ERROR_021, XmlErrorFormat.MonitoringReportListError, packageName);
-            }
-            if (signalService.isPackagesByStatusExist(username, packageName, Status.DELETED)) {
-                throw clientExcp(SignalError.ERROR_022, XmlErrorFormat.MonitoringReportListError, packageName);
-            }
-            if (!signalService.isPackagesByStatusExist(username, packageName, Status.ON_MONITORING)) {
-                throw clientExcp(SignalError.ERROR_012, XmlErrorFormat.MonitoringReportListError, packageName);
-            }
-
-            String memberCode = username.substring(0, 6);
-            String filePrefix = (memberCode + "_" + packageName).toUpperCase();
-            String dirPath = config.getLoadDir() + File.separator + memberCode;
-            File folder = new File(dirPath);
-
-            MonitoringReportListResponse.Reports reports = new MonitoringReportListResponse.Reports();
-            if (folder.exists() && folder.isDirectory()) {
-                File[] files = folder.listFiles();
-                if (files != null) {
-                    for (File file : files) {
-                        if (file.isFile() && file.getName().startsWith(filePrefix)) {
-                            reports.getReports().add(file.getName());
-                        }
-                    }
-                }
-            }
-            MonitoringReportListResponse response = new MonitoringReportListResponse();
-            response.setPackageName(packageName);
-            response.setReports(reports);
-            return ResponseEntity.ok(response);
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringReportListError, e);
-        }
-    }
-
-    public ResponseEntity<Resource> downloadFile(String fileName) {
-        try {
-            String memberCode = CTUtil.getMemberCodeFromFileName(fileName);
-            if (memberCode == null) {
-                throw clientExcp(SignalError.ERROR_014, XmlErrorFormat.MonitoringDownloadError, fileName);
-            }
-            String dirPath = config.getLoadDir() + File.separator + memberCode;
-            File folder = new File(dirPath);
-
-            if (folder.exists() && folder.isDirectory()) {
-                File file = new File(dirPath + File.separator + fileName);
-                if (file.exists() && file.isFile()) {
-                    return ResponseEntity.ok().body(new FileUrlResource(file.getPath()));
-                }
-            } else {
-                throw clientExcp(SignalError.ERROR_014, XmlErrorFormat.MonitoringDownloadError, fileName);
-            }
-            throw clientExcp(SignalError.ERROR_014, XmlErrorFormat.MonitoringDownloadError, fileName);
-        } catch (Exception e) {
-            if (e instanceof SignalClientException) {
-                throw (SignalClientException) e;
-            }
-            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringDownloadError, e);
-        }
-    }
-
-    private SignalFacade getCurrentProxy() {
-        if (self == null) {
-            self = (SignalFacade) AopContext.currentProxy();
-        }
-        return self;
-    }
-}

src/main/java/ru/nbch/credit_tracker/facade/SignalFacadeV3.java

@@ -0,0 +1,366 @@
+package ru.nbch.credit_tracker.facade;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.xml.stream.XMLStreamException;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.FileUrlResource;
+import org.springframework.core.io.Resource;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+import ru.nbch.credit_tracker.component.MonitoringRequestValidator;
+import ru.nbch.credit_tracker.component.comparator.DeletePackageStatusComparator;
+import ru.nbch.credit_tracker.component.comparator.ReportListPackageStatusComparator;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.entities.user_request.active_packages.MonitoringActivePackagesRequest;
+import ru.nbch.credit_tracker.entities.user_request.all_packages.MonitoringAllPackagesRequest;
+import ru.nbch.credit_tracker.entities.user_response.active_packages.MonitoringActivePackagesResponse;
+import ru.nbch.credit_tracker.entities.user_response.active_packages.Package;
+import ru.nbch.credit_tracker.entities.user_response.active_packages.Packages;
+import ru.nbch.credit_tracker.entities.user_response.all_packages.MonitoringAllPackagesResponse;
+import ru.nbch.credit_tracker.entities.user_response.all_packages.PackageAll;
+import ru.nbch.credit_tracker.entities.user_response.all_packages.PackagesAll;
+import ru.nbch.credit_tracker.entities.user_response.success.MonitoringPackageDeleteResponse;
+import ru.nbch.credit_tracker.entities.user_response.success.MonitoringReportListResponse;
+import ru.nbch.credit_tracker.enums.SignalError;
+import ru.nbch.credit_tracker.enums.Status;
+import ru.nbch.credit_tracker.enums.XmlErrorFormat;
+import ru.nbch.credit_tracker.exceptions.SignalClientException;
+import ru.nbch.credit_tracker.model.MonitoringValidationError;
+import ru.nbch.credit_tracker.model.ValidationResult;
+import ru.nbch.credit_tracker.service.UserPackageService;
+import ru.nbch.credit_tracker.service.scoring.AuthorizationService;
+import ru.nbch.credit_tracker.service.signal.SignalRestService;
+import ru.nbch.credit_tracker.service.task.Pipeline;
+import ru.nbch.credit_tracker.service.task.PipelinePackageFabric;
+import ru.nbch.credit_tracker.service.task.PipelinePackageOrchestrator;
+import ru.nbch.credit_tracker.service.task.SetupParam;
+import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
+import ru.nbch.credit_tracker.service.task.impl.Saver;
+import ru.nbch.credit_tracker.utils.CtFileUtils;
+import ru.nbch.credit_tracker.utils.EnumUtil;
+import ru.nbch.credit_tracker.utils.LogUtil;
+import ru.nbch.credit_tracker.utils.ZipUtils;
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.clientExcp;
+import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.serverExcp;
+
+@Service
+@Slf4j
+public class SignalFacadeV3 {
+
+    private final MonitoringRequestValidator monitoringRequestValidator;
+    private final CreditTrackerSettings config;
+    private final UserPackageService userPackageService;
+    private final AuthorizationService authorizationService;
+    private final PipelinePackageFabric pipelinePackageManager;
+    private final SignalRestService signalRestService;
+    private final Saver saver;
+    private final ExecutorService pipelineExecutorService;
+    private final ExecutorService saverExecutorService;
+    private final PipelinePackageOrchestrator pipelineOrchestrator;
+    private final Pattern DOWNLOAD_FILE_PATTERN = Pattern.compile("^(?<member>[A-Za-z0-9]{6})_(?<package>[A-Za-z0-9_]+)_(?<ts>\\d{4}\\d{2}\\d{2}_\\d{2}\\d{2}\\d{2})(?:\\.xml)?\\.zip$");
+
+    public SignalFacadeV3(MonitoringRequestValidator monitoringRequestValidator,
+                          CreditTrackerSettings config,
+                          UserPackageService userPackageService,
+                          AuthorizationService authorizationService,
+                          PipelinePackageFabric pipelinePackageManager,
+                          SignalRestService signalRestService,
+                          Saver saver,
+                          ExecutorService pipelineExecutorService,
+                          ExecutorService saverExecutorService, PipelinePackageOrchestrator pipelineOrchestrator) {
+        this.monitoringRequestValidator = monitoringRequestValidator;
+        this.config = config;
+        this.userPackageService = userPackageService;
+        this.authorizationService = authorizationService;
+        this.pipelinePackageManager = pipelinePackageManager;
+        this.signalRestService = signalRestService;
+        this.saverExecutorService = saverExecutorService;
+        this.pipelineExecutorService = pipelineExecutorService;
+        this.saver = saver;
+        this.pipelineOrchestrator = pipelineOrchestrator;
+    }
+
+    public void setupMonitoringRequest(MultipartFile file) {
+        log.debug("Start processing monitoring request");
+        monitoringRequestValidator.validateZipContent(file);
+
+        log.trace("Saving monitoring request to tmp file...");
+        Path tmpReqFilePath;
+        try {
+            String folderPath = config.getLoadDir() + File.separator + "tmp" + File.separator;
+            tmpReqFilePath = ZipUtils.unzipAndGetFirstEntry(folderPath, file);
+        } catch (IOException e) {
+            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
+        }
+
+        log.debug("File saved to {}", tmpReqFilePath);
+        ValidationResult validationResult = null;
+        try {
+            validationResult = monitoringRequestValidator.readAndValidateMonitoringRequest(tmpReqFilePath);
+            if (!validationResult.getValidationErrors().isEmpty()) {
+                MonitoringValidationError firstError = validationResult.getValidationErrors().getFirst();
+                userPackageService.rejectPackage(validationResult.getUserId(), validationResult.getPackageName(), validationResult.getSignalTypes());
+                Files.delete(tmpReqFilePath);
+                throw firstError.getSignalException();
+            }
+        } catch (RuntimeException e) {
+            if (e.getCause() instanceof XMLStreamException) {
+                throw clientExcp(SignalError.ERROR_011, XmlErrorFormat.MonitoringError);
+            }
+            if (e instanceof SignalClientException) {
+                throw e;
+            }
+            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
+        } catch (IOException e) {
+            log.warn("Error deleting tmp file", e);
+        }
+
+        String packageName = validationResult.getPackageName();
+        String userId = validationResult.getUserId();
+        String password = validationResult.getPassword();
+        List<Integer> signalTypes = validationResult.getSignalTypes();
+
+        Path parsedFilePath = tmpReqFilePath.resolveSibling(Path.of(userId.substring(0, 6) + "_" + packageName + ".xml"));
+        try {
+            Files.move(tmpReqFilePath, parsedFilePath, StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            // проверка авторизации
+            AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(userId, password);
+            if (!authResult.isSuccess()) {
+                userPackageService.rejectPackage(userId, packageName, signalTypes);
+                throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringError);
+            }
+
+            // проверка на уникальность пакета
+            if (userPackageService.isPackageNotUnique(userId, packageName)) {
+                userPackageService.rejectPackage(userId, packageName, signalTypes);
+                throw clientExcp(SignalError.ERROR_007, XmlErrorFormat.MonitoringError);
+            }
+
+            Long packageId = userPackageService.registerPackage(userId, packageName, signalTypes);
+            LogUtil.addMdcLogKey("%s %d".formatted(packageName, packageId));
+            log.info("Package registered. PackageId {}. Status: {}", packageId, Status.ACCEPTED.name());
+            Pipeline pipeline = pipelinePackageManager.build(new SetupParam(packageId, parsedFilePath, PipelineMode.DEFAULT));
+            pipelineOrchestrator.startOrDelay(() -> {
+                saver.readAndSave(packageId, parsedFilePath);
+                CtFileUtils.deleteQuietly(parsedFilePath);
+            }, pipeline, packageId);
+        } catch (Exception e) {
+            try {
+                Files.delete(parsedFilePath);
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+            if (e instanceof SignalClientException) {
+                throw (SignalClientException) e;
+            }
+            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringError, e);
+        }
+    }
+
+    public ResponseEntity<MonitoringActivePackagesResponse> findActivePackages(MonitoringActivePackagesRequest activePackagesRequest) throws Exception {
+        AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(activePackagesRequest.getUserId(), activePackagesRequest.getPassword());
+        if (!authResult.isSuccess()) {
+            throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringActivePackagesError);
+        }
+
+        List<UserPackage> activePackages = userPackageService.findActivePackages(activePackagesRequest.getUserId());
+        if (activePackages.isEmpty()) {
+            throw clientExcp(SignalError.ERROR_015, XmlErrorFormat.MonitoringActivePackagesError);
+        }
+
+        MonitoringActivePackagesResponse response = new MonitoringActivePackagesResponse();
+        response.setStatus("OK");
+        response.setMembercode(activePackagesRequest.getUserId().substring(0, 6));
+        Packages packages = new Packages();
+        activePackages.forEach(data -> {
+            Package pack = new Package();
+            pack.setPackageName(data.getPackageName());
+            pack.setCreatedAt(data.getCreatedAt() != null ? data.getCreatedAt().toString() : null);
+            packages.getPackages().add(pack);
+        });
+        response.setPackages(packages);
+        return ResponseEntity.ok(response);
+    }
+
+    public ResponseEntity<MonitoringAllPackagesResponse> findAllPackages(MonitoringAllPackagesRequest allPackagesRequest) throws Exception {
+        AuthorizationService.AuthResult authResult = authorizationService.findAuthInfo(allPackagesRequest.getUserId(), allPackagesRequest.getPassword());
+        if (!authResult.isSuccess()) {
+            throw clientExcp(authResult.getError(), XmlErrorFormat.MonitoringAllPackagesError);
+        }
+
+        List<UserPackage> activePackages = userPackageService.findAllPackages(allPackagesRequest.getUserId());
+        if (activePackages.isEmpty()) {
+            throw clientExcp(SignalError.ERROR_019, XmlErrorFormat.MonitoringAllPackagesError);
+        }
+
+        MonitoringAllPackagesResponse response = new MonitoringAllPackagesResponse();
+        response.setStatus("OK");
+        response.setMembercode(allPackagesRequest.getUserId().substring(0, 6));
+        PackagesAll packages = new PackagesAll();
+        activePackages.forEach(data -> {
+            PackageAll pack = new PackageAll();
+            pack.setPackageName(data.getPackageName());
+            pack.setCreatedAt(data.getCreatedAt() != null ? data.getCreatedAt().toString() : null);
+            Status packageStatus = EnumUtil.getEnum(Status.class, data.getStatus());
+            Status statusForClient = packageStatus == Status.ACCEPTED ? Status.PROCESSING :
+                Arrays.asList(Status.PROCESSING_REPEATED, Status.NO_FID).contains(packageStatus) ? Status.ON_MONITORING : packageStatus;
+            assert statusForClient != null;
+            pack.setStatus(statusForClient.name());
+            if (packageStatus == Status.DELETED) {
+                pack.setDeactivatedAt(data.getDeactivatedAt() != null ? data.getDeactivatedAt().toString() : null);
+            }
+            packages.getPackages().add(pack);
+        });
+        response.setPackages(packages);
+        return ResponseEntity.ok(response);
+    }
+
+    public ResponseEntity<MonitoringPackageDeleteResponse> deletePackage(String username, String packageName) {
+        UserPackage packageToDelete = userPackageService.findByMemberCodeAndPackageName(username, packageName)
+            .stream()
+            .max(DeletePackageStatusComparator.INSTANCE)
+            .orElseThrow(()
+                -> clientExcp(SignalError.ERROR_016, XmlErrorFormat.MonitoringPackageDeleteError, packageName));
+
+        Status status = EnumUtil.getEnum(Status.class, packageToDelete.getStatus());
+        if (!Arrays.asList(Status.ON_MONITORING, Status.NO_FID).contains(status)) {
+            throw clientExcp(deleteStatusError(status), XmlErrorFormat.MonitoringPackageDeleteError, packageName);
+        }
+
+        if (status == Status.ON_MONITORING) {
+            boolean successfullyDeleted = signalRestService.deletePackage(username, packageName);
+            if (!successfullyDeleted) {
+                throw clientExcp(SignalError.ERROR_017, XmlErrorFormat.MonitoringPackageDeleteError, packageName);
+            }
+        }
+
+        LocalDateTime deactivatedAt = LocalDateTime.now();
+        userPackageService.deletePackageById(packageToDelete.getId(), deactivatedAt);
+        MonitoringPackageDeleteResponse response = new MonitoringPackageDeleteResponse();
+        response.setStatus("Deleted");
+        response.setPackageName(packageName);
+        response.setDeactivatedAt(deactivatedAt.toString());
+
+        log.info("Package {} deleted", packageName);
+        return ResponseEntity.ok(response);
+    }
+
+
+    private SignalError deleteStatusError(Status status) {
+        if (status == null) return SignalError.ERROR_017;
+        return switch (status) {
+            case PROCESSING, ACCEPTED -> SignalError.ERROR_023;
+            case REJECTED, ERROR -> SignalError.ERROR_024;
+            case DELETED -> SignalError.ERROR_025;
+            case PROCESSING_REPEATED -> SignalError.ERROR_026;
+            default -> throw new IllegalStateException("Unexpected value: " + status);
+        };
+    }
+
+    public ResponseEntity<MonitoringReportListResponse> getReportList(String username, String packageName) {
+        try {
+            UserPackage userPackage = userPackageService
+                .findByMemberCodeAndPackageName(username, packageName)
+                .stream()
+                .max(ReportListPackageStatusComparator.INSTANCE)
+                .orElse(null);
+            if (userPackage == null) {
+                throw clientExcp(SignalError.ERROR_012, XmlErrorFormat.MonitoringReportListError, packageName);
+            }
+            Status status = EnumUtil.getEnum(Status.class, userPackage.getStatus());
+            if (status == null) {
+                throw clientExcp(SignalError.ERROR_017, XmlErrorFormat.MonitoringReportListError);
+            }
+            if (Arrays.asList(Status.PROCESSING, Status.ACCEPTED).contains(status)) {
+                throw clientExcp(SignalError.ERROR_018, XmlErrorFormat.MonitoringReportListError, packageName);
+            }
+            if (status.equals(Status.REJECTED)) {
+                throw clientExcp(SignalError.ERROR_020, XmlErrorFormat.MonitoringReportListError, packageName);
+            }
+            if (status.equals(Status.ERROR)) {
+                throw clientExcp(SignalError.ERROR_021, XmlErrorFormat.MonitoringReportListError, packageName);
+            }
+            if (status.equals(Status.DELETED)) {
+                throw clientExcp(SignalError.ERROR_022, XmlErrorFormat.MonitoringReportListError, packageName);
+            }
+            if (!ReportListPackageStatusComparator.reportedStatuses.contains(status)) {
+                throw clientExcp(SignalError.ERROR_012, XmlErrorFormat.MonitoringReportListError, packageName);
+            }
+
+            String memberCode = username.substring(0, 6);
+            String filePrefix = (memberCode + "_" + packageName).toUpperCase();
+            String dirPath = config.getLoadDir() + File.separator + memberCode;
+            File folder = new File(dirPath);
+            log.debug("searching for reports from: {}", folder.getAbsolutePath());
+
+            MonitoringReportListResponse.Reports reports = new MonitoringReportListResponse.Reports();
+            if (folder.exists() && folder.isDirectory()) {
+                File[] files = folder.listFiles();
+                if (files != null) {
+                    for (File file : files) {
+                        if (file.isFile() && file.getName().startsWith(filePrefix)) {
+                            reports.getReports().add(file.getName());
+                        }
+                    }
+                }
+            }
+            MonitoringReportListResponse response = new MonitoringReportListResponse();
+            response.setPackageName(packageName);
+            response.setReports(reports);
+            return ResponseEntity.ok(response);
+        } catch (Exception e) {
+            if (e instanceof SignalClientException) {
+                throw (SignalClientException) e;
+            }
+            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringReportListError, e);
+        }
+    }
+
+    public ResponseEntity<Resource> downloadFile(String fileName) {
+        try {
+            Matcher matcher = DOWNLOAD_FILE_PATTERN.matcher(fileName);
+            String memberCode = null;
+            if (matcher.matches()) {
+                memberCode = matcher.group("member");
+            }
+
+            if (memberCode == null) {
+                throw clientExcp(SignalError.ERROR_014, XmlErrorFormat.MonitoringDownloadError, fileName);
+            }
+            Path reportRootPath = Path.of(config.getLoadDir(), memberCode);
+            File folder = reportRootPath.toFile();
+
+            if (folder.exists() && folder.isDirectory()) {
+                Path filePath = reportRootPath.resolve(fileName);
+                File file = filePath.toFile();
+                if (file.exists() && file.isFile()) {
+                    return ResponseEntity.ok().body(new FileUrlResource(file.getPath()));
+                }
+            } else {
+                throw clientExcp(SignalError.ERROR_014, XmlErrorFormat.MonitoringDownloadError, fileName);
+            }
+            throw clientExcp(SignalError.ERROR_014, XmlErrorFormat.MonitoringDownloadError, fileName);
+        } catch (Exception e) {
+            if (e instanceof SignalClientException) {
+                throw (SignalClientException) e;
+            }
+            throw serverExcp(SignalError.ERROR_013, XmlErrorFormat.MonitoringDownloadError, e);
+        }
+    }
+}

src/main/java/ru/nbch/credit_tracker/mapper/indic/FlagsMapper.java

@@ -1,19 +0,0 @@
-package ru.nbch.credit_tracker.mapper.indic;
-
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.service.indic.FlagsService;
-
-import java.time.LocalDate;
-import java.util.List;
-
-@Mapper
-public interface FlagsMapper {
-
-    List<FlagsService.FlagFidData>  findFlagsDebt5000(@Param("fids") List<Integer> fids);
-
-    List<FlagsService.FlagFidData>  findFlag90ByPastDueArrear(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
-
-    List<FlagsService.FlagFidData>  findFlag90ByRuPaymt(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
-
-}

src/main/java/ru/nbch/credit_tracker/mapper/indic/IndicFlagsMapper.java

@@ -0,0 +1,19 @@
+package ru.nbch.credit_tracker.mapper.indic;
+
+import java.time.LocalDate;
+import java.util.List;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.indic.PaymentsFlag;
+
+@Mapper
+public interface IndicFlagsMapper {
+    List<PaymentsFlag> findFlagsDebt5000(@Param("fids") List<Integer> fids);
+
+    List<PaymentsFlag> findFlag90ByPastDueArrear(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
+
+    List<PaymentsFlag> findFlag90ByRuPaymt(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
+
+    List<PaymentsFlag> findFlag90ByRuAllArrears(@Param("fids") List<Integer> fids, @Param("reportedDate") LocalDate reportedDate);
+
+}

src/main/java/ru/nbch/credit_tracker/mapper/indic/PersonMapper.java

@@ -1,15 +1,16 @@
 package ru.nbch.credit_tracker.mapper.indic;
 
+import java.util.List;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.service.indic.PersonService;
-
-import java.util.List;
+import ru.nbch.credit_tracker.entities.indic.Id;
+import ru.nbch.credit_tracker.entities.indic.Name;
 
 @Mapper
 public interface PersonMapper {
 
-    List<PersonService.NameData> findNameData(@Param("fids") List<Integer> fids);
+    List<Name> findNameData(@Param("phones") List<String> phones);
+    List<Name> findNameDataByFids(@Param("fids") List<Integer> fids);
 
-    List<PersonService.PassportData> findPassportData(@Param("fids") List<Integer> fids);
+    List<Id> findPassportData(@Param("fids") List<Integer> fids);
 }

src/main/java/ru/nbch/credit_tracker/mapper/indic/PhoneMapper.java

@@ -1,12 +0,0 @@
-package ru.nbch.credit_tracker.mapper.indic;
-
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.service.indic.PhoneService;
-
-import java.util.List;
-
-@Mapper
-public interface PhoneMapper {
-    List<PhoneService.FidsData> findFids(@Param("phones") List<String> phones);
-}

src/main/java/ru/nbch/credit_tracker/mapper/signal/AppVersionMapper.java

@@ -0,0 +1,9 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import org.apache.ibatis.annotations.Mapper;
+import ru.nbch.credit_tracker.entities.app.AppVersion;
+
+@Mapper
+public interface AppVersionMapper {
+    AppVersion getAppVersion();
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/FlagDetailMapper.java

@@ -0,0 +1,14 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import java.util.List;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.FlagDetail;
+
+@Mapper
+public interface FlagDetailMapper {
+    void registerCalcFlags(@Param("json") String json);
+    void updateCalcFlags(@Param("json") String json);
+    FlagDetail getByMapId(@Param("mapId") Long mapId);
+    void deleteFlags(@Param("mapIds") List<Long> mapIds);
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/HitsMapper.java

@@ -2,7 +2,7 @@ package ru.nbch.credit_tracker.mapper.signal;
 
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.entities.dto.Hit;
+import ru.nbch.credit_tracker.entities.app.Hit;
 
 import java.util.List;
 

src/main/java/ru/nbch/credit_tracker/mapper/signal/PackagesRecordsJoinMapper.java

@@ -0,0 +1,11 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import java.util.List;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.PackageRecordWithPhone;
+
+@Mapper
+public interface PackagesRecordsJoinMapper {
+    List<PackageRecordWithPhone> findPackagesRecordsJoinPage(@Param("packageId") Long packageId, @Param("limit") Integer limit, @Param("offset") Integer offset);
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/PackagesRecordsMapper.java

@@ -0,0 +1,17 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import java.util.List;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.PackagesRecordsMap;
+
+@Mapper
+public interface PackagesRecordsMapper {
+    void setErrorJson(@Param("json") String json);
+    List<Long> registerJson(@Param("json") String json);
+    List<PackagesRecordsMap> findFullPhoneFidMap(@Param("packageId") Long packageId, @Param("limit") int limit, @Param("offset") int offset);
+    List<Long> uniquePackageIdByPhones(@Param("json") String json);
+    Long deleteByPackageId(@Param("packageId") Long id);
+
+    PackagesRecordsMap findById(Long id);
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/PhoneFidMapMapper.java

@@ -1,24 +0,0 @@
-package ru.nbch.credit_tracker.mapper.signal;
-
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.user_request.monitoring_request.Subject;
-
-import java.util.List;
-
-@Mapper
-public interface PhoneFidMapMapper {
-
-    void registerSubjects(@Param("subjects") List<Subject> subjects, @Param("packageId") Long packageId);
-
-    List<PhoneFidMap> findFullPhoneFidMap(@Param("packageId") Long packageId, @Param("offset") int offset, @Param("limit") int limit);
-
-    void updatePhoneFidMaps(@Param("subjects") List<PhoneFidMap> subjects);
-
-    void updatePhoneFidMapsErrors(@Param("subjects") List<PhoneFidMap> subjects);
-
-    PhoneFidMap findPhoneFidMapById(@Param("id") Long packageId);
-
-    boolean hasNonNullFids(@Param("packageId") Long packageId);
-}

src/main/java/ru/nbch/credit_tracker/mapper/signal/PhonesMapMapper.java

@@ -0,0 +1,26 @@
+package ru.nbch.credit_tracker.mapper.signal;
+
+import java.util.List;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.PhonesMap;
+
+@Mapper
+public interface PhonesMapMapper {
+    List<PhonesMap> getCachedData(@Param("phones") List<Long> phones);
+    List<PhonesMap> searchedAtLessThan(@Param("daysCount") Integer daysCount, @Param("batchSize") int batchSize, @Param("lastId") Long lastId);
+    PhonesMap findByPhone(@Param("phone") Long phone);
+    List<PhonesMap> findEligibleForFlagsUpdate(@Param("lastId") Long lastId,
+                                               @Param("interval") Integer interval,
+                                               @Param("batchSize") Integer batchSize);
+    List<PhonesMap> findEligibleForFlagsInsert(@Param("lastId") Long lastId,
+                                               @Param("batchSize") Integer batchSize);
+    void updateIdCachedData(@Param("json") String json);
+    void updateNameCachedData(@Param("json") String json);
+    void updateFlagCachedData(@Param("json") String json);
+    void resetFidAndPD(@Param("json") String json);
+    void updateSearchedAt(@Param("json") String json);
+    void registerJson(@Param("json") String json);
+    void setFidJson(@Param("json") String json);
+    void deleteUnusedPhones();
+}

src/main/java/ru/nbch/credit_tracker/mapper/signal/UserPackageMapper.java

@@ -1,22 +1,17 @@
 package ru.nbch.credit_tracker.mapper.signal;
 
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-
 import java.time.LocalDateTime;
 import java.util.List;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
 
 @Mapper
 public interface UserPackageMapper {
 
     boolean isPackageNotUnique(@Param("userId") String userId, @Param("packageName") String packageName);
 
-    boolean isPackageExists(@Param("userId") String userId, @Param("packageName") String packageName, @Param("status") String status);
-
-    boolean hasActivePackages(@Param("userId") String userId, @Param("packageName")  String packageName, @Param("statuses") List<String> statuses);
-
-    void deletePackage(@Param("userId") String userId, @Param("packageName") String packageName, @Param("status") String newStatus, @Param("previousStatus") String previousStatus, @Param("deactivatedAt") LocalDateTime deactivatedAt);
+    void deletePackageById(@Param("id") Long id, @Param("status") String newStatus, @Param("deactivatedAt") LocalDateTime deactivatedAt);
 
     Long insertPackage(@Param("userPackage") UserPackage userPackage);
 
@@ -25,12 +20,22 @@ public interface UserPackageMapper {
     void updatePackageLastDownloadedReport(@Param("id") Long id, @Param("fileName") String fileName);
 
     void updatePackageOnMonitoringSuccess(@Param("id") Long id, @Param("monitoringAt") LocalDateTime monitoringAt);
+    void updateNoFidMonitoring(@Param("id") Long id);
+    void setFidChanged(@Param("packageIds") List<Long> id);
 
     UserPackage findPackage(@Param("id") Long id);
+    List<UserPackage> findForRepeatedPackages();
 
     List<UserPackage> findUserPackages(@Param("userId") String userId, @Param("statuses") List<String> status);
 
-    List<UserPackage> findAllPackagesByStatus(@Param("status")  String status);
+    List<UserPackage> findAllPackagesByStatus(@Param("statuses") String... statuses);
+    List<UserPackage> findOnMonitoringByMemberCode(List<String> membercodes);
 
     List<UserPackage> findAllPackagesByUserId(@Param("userId") String userId);
+    UserPackage findByPackageName(@Param("packageName") String packageName);
+    List<UserPackage> findOnMonitoringPackages();
+    List<UserPackage> findByMemberCodeAndPackageName(@Param("memberCode") String memberCode,
+                                                     @Param("packageName") String packageName);
+
+    void setRejectedStatus(@Param("id") Long id, @Param("status") String status);
 }

src/main/java/ru/nbch/credit_tracker/model/FlagData.java

@@ -0,0 +1,15 @@
+package ru.nbch.credit_tracker.model;
+
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+public class FlagData {
+    private Long phone;
+    private Integer flag_debt5000;
+    private Integer flag_90_12;
+}

src/main/java/ru/nbch/credit_tracker/model/IdData.java

@@ -0,0 +1,25 @@
+package ru.nbch.credit_tracker.model;
+
+
+import java.time.LocalDate;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class IdData {
+    private Long phone;
+    private Integer fid;
+    private String serNum;
+    private String idNum;
+    private LocalDate issDate;
+    private String idType;
+    private String idType2;
+    private LocalDate fileSinceDate;
+}

src/main/java/ru/nbch/credit_tracker/model/MonitoringValidationError.java

@@ -0,0 +1,30 @@
+package ru.nbch.credit_tracker.model;
+
+import ru.nbch.credit_tracker.exceptions.SignalException;
+
+public class MonitoringValidationError {
+    private final int line;
+    private final String message;
+    private final SignalException signalException;
+
+
+    public MonitoringValidationError(int line,
+                                     String message,
+                                     SignalException signalException) {
+        this.line = line;
+        this.message = message;
+        this.signalException = signalException;
+    }
+
+    public int getLine() {
+        return line;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public SignalException getSignalException() {
+        return signalException;
+    }
+}

src/main/java/ru/nbch/credit_tracker/model/NameData.java

@@ -0,0 +1,19 @@
+package ru.nbch.credit_tracker.model;
+
+
+import java.time.LocalDate;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+public class NameData {
+    private Integer fid;
+    private String firstName;
+    private String lastName;
+    private String middleName;
+    private LocalDate birthDate;
+    private LocalDate fileSinceDate;
+}

src/main/java/ru/nbch/credit_tracker/model/PipelineContext.java

@@ -0,0 +1,21 @@
+package ru.nbch.credit_tracker.model;
+
+import java.util.EnumMap;
+
+public class PipelineContext<T extends Enum<T>> {
+    private final EnumMap<T, Object> storage;
+
+    public PipelineContext(Class<T> keyClass) {
+        this.storage = new EnumMap<>(keyClass);
+    }
+
+    public void store(T key, Object value) {
+        storage.put(key, value);
+    }
+
+    @SuppressWarnings("unchecked")
+    public <S> S get(T key) {
+        Object o = storage.get(key);
+        return (S) o;
+    }
+}

src/main/java/ru/nbch/credit_tracker/model/StagePayload.java

@@ -0,0 +1,7 @@
+package ru.nbch.credit_tracker.model;
+
+public interface StagePayload {
+    boolean isPoison();
+    default void setContext(PipelineContext<?> context) {}
+    default PipelineContext<?> getContext() {return null;}
+}

src/main/java/ru/nbch/credit_tracker/model/SubjectProfile.java

@@ -0,0 +1,18 @@
+package ru.nbch.credit_tracker.model;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Builder
+public class SubjectProfile {
+    private String identifier;
+    private Long mapId;
+    private Long phone;
+    private Integer fid;
+    private NameData nameData;
+    private IdData idData;
+    private FlagData flagData;
+}

src/main/java/ru/nbch/credit_tracker/model/TerminatePayload.java

@@ -0,0 +1,4 @@
+package ru.nbch.credit_tracker.model;
+
+public interface TerminatePayload {
+}

src/main/java/ru/nbch/credit_tracker/model/ValidationResult.java

@@ -0,0 +1,60 @@
+package ru.nbch.credit_tracker.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ValidationResult {
+    private String packageName;
+    private String userId;
+    private String password;
+    private List<Integer> signalTypes = new ArrayList<>();
+    private List<MonitoringValidationError> validationErrors = new ArrayList<>();
+
+    public void setPackageName(String packageName) {
+        this.packageName = packageName;
+    }
+
+    public void setUserId(String userId) {
+        this.userId = userId;
+    }
+
+    public void setValidationErrors(List<MonitoringValidationError> validationErrors) {
+        this.validationErrors = validationErrors;
+    }
+
+    public ValidationResult() {
+    }
+
+    public String getPackageName() {
+        return packageName;
+    }
+
+    public String getUserId() {
+        return userId;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public List<Integer> getSignalTypes() {
+        return signalTypes;
+    }
+
+    public void setSignalTypes(List<Integer> signalTypes) {
+        this.signalTypes = signalTypes;
+    }
+
+    public List<MonitoringValidationError> getValidationErrors() {
+        return validationErrors;
+    }
+
+    public void  addError(MonitoringValidationError error) {
+        this.validationErrors.add(error);
+    }
+}
+

src/main/java/ru/nbch/credit_tracker/model/impl/EmptyTerminatePayload.java

@@ -0,0 +1,6 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import ru.nbch.credit_tracker.model.TerminatePayload;
+
+public class EmptyTerminatePayload implements TerminatePayload {
+}

src/main/java/ru/nbch/credit_tracker/model/impl/PackageVariable.java

@@ -0,0 +1,8 @@
+package ru.nbch.credit_tracker.model.impl;
+
+public enum PackageVariable {
+    packageId,
+    cacheMisses,
+    cacheHits,
+    mode
+}

src/main/java/ru/nbch/credit_tracker/model/impl/PartFilePayload.java

@@ -0,0 +1,37 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import java.nio.file.Path;
+import lombok.Getter;
+import ru.nbch.credit_tracker.enums.PoisonType;
+import ru.nbch.credit_tracker.model.StagePayload;
+
+@Getter
+public class PartFilePayload implements StagePayload {
+    private final Path partPath;
+    private final Long packageId;
+    private final String membercode;
+    private final String packageName;
+    private final PoisonType poison;
+    public static final PartFilePayload POISON = new PartFilePayload(PoisonType.NORMALLY);
+
+    public PartFilePayload(PoisonType poison) {
+        this.poison = poison;
+        this.membercode = null;
+        this.packageName = null;
+        this.partPath = null;
+        this.packageId = null;
+    }
+
+    public PartFilePayload(Path partPath, Long packageId, String membercode, String packageName) {
+        this.partPath = partPath;
+        this.packageId = packageId;
+        this.membercode = membercode;
+        this.packageName = packageName;
+        this.poison = PoisonType.NONE;
+    }
+
+    @Override
+    public boolean isPoison() {
+        return poison == PoisonType.NORMALLY;
+    }
+}

src/main/java/ru/nbch/credit_tracker/model/impl/SignalXmlFilePayload.java

@@ -0,0 +1,7 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import java.nio.file.Path;
+import ru.nbch.credit_tracker.model.TerminatePayload;
+
+public record SignalXmlFilePayload(Path signalXmlFilePath) implements TerminatePayload {
+}

src/main/java/ru/nbch/credit_tracker/model/impl/SubjectPayload.java

@@ -0,0 +1,65 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import java.util.List;
+import ru.nbch.credit_tracker.enums.PoisonType;
+import ru.nbch.credit_tracker.model.PipelineContext;
+import ru.nbch.credit_tracker.model.StagePayload;
+import ru.nbch.credit_tracker.model.SubjectProfile;
+
+
+public class SubjectPayload implements StagePayload {
+    private final Long packageId;
+    private final String memberCode;
+    private final String packageName;
+    private final List<SubjectProfile> subjects;
+    private PipelineContext<PackageVariable> ctx;
+    public static final SubjectPayload POISON = new SubjectPayload(null, PoisonType.NORMALLY);
+    private final PoisonType poison;
+
+    public SubjectPayload(Long packageId, PoisonType poison) {
+        this.memberCode = null;
+        this.packageName = null;
+        this.packageId = packageId;
+        this.subjects = null;
+        this.poison = poison;
+    }
+
+    public SubjectPayload(Long packageId, String memberCode, String packageName, List<SubjectProfile> subjects) {
+        this.packageId = packageId;
+        this.memberCode = memberCode;
+        this.packageName = packageName;
+        this.subjects = subjects;
+        this.poison = PoisonType.NONE;
+    }
+
+    @Override
+    public void setContext(PipelineContext<?> context) {
+        this.ctx = (PipelineContext<PackageVariable>) context;
+    }
+
+    @Override
+    public PipelineContext<?> getContext() {
+        return this.ctx;
+    }
+
+    public Long getPackageId() {
+        return packageId;
+    }
+
+    public List<SubjectProfile> getSubjects() {
+        return subjects;
+    }
+
+    public String getMemberCode() {
+        return memberCode;
+    }
+
+    public String getPackageName() {
+        return packageName;
+    }
+
+    @Override
+    public boolean isPoison() {
+        return poison == PoisonType.NORMALLY;
+    }
+}

src/main/java/ru/nbch/credit_tracker/model/impl/VoidPayload.java

@@ -0,0 +1,24 @@
+package ru.nbch.credit_tracker.model.impl;
+
+import lombok.Getter;
+import ru.nbch.credit_tracker.enums.PoisonType;
+import ru.nbch.credit_tracker.model.StagePayload;
+
+@Getter
+public class VoidPayload implements StagePayload {
+    private final PoisonType poison;
+    public static final VoidPayload POISON = new VoidPayload(PoisonType.NORMALLY);
+
+    public VoidPayload(PoisonType poison) {
+        this.poison = poison;
+    }
+
+    public VoidPayload() {
+        this.poison = PoisonType.NONE;
+    }
+
+    @Override
+    public boolean isPoison() {
+        return poison == PoisonType.NORMALLY;
+    }
+}

src/main/java/ru/nbch/credit_tracker/repository/PhoneRepository.java

@@ -0,0 +1,159 @@
+package ru.nbch.credit_tracker.repository;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.time.LocalDate;
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.stereotype.Repository;
+import ru.nbch.credit_tracker.entities.indic.Name;
+
+@Repository
+public class PhoneRepository {
+
+    private static final String SELCT_FIDS = """
+        SELECT
+            pn.number,
+            pn.fid,
+            pn.file_since_dt
+        FROM PHONE_NORM pn
+            JOIN (SELECT DISTINCT fid FROM NAME) nf
+            ON nf.fid = pn.fid
+        WHERE number in (:phones)
+        ORDER BY
+            pn.number ASC,
+            pn.file_since_dt DESC,
+            pn.fid DESC
+        WITH UR
+        """;
+
+    private static final String SELECT_FIDS_WITH_NAME = """
+        SELECT
+            pn.number,
+            pn.fid,
+            pn.file_since_dt,
+            nf.first,
+            nf.middle,
+            nf.name_1,
+            nf.birth_dt,
+            nf.file_since_dt as file_since_dt_name
+        FROM PHONE_NORM pn
+                 JOIN NAME nf
+                      ON nf.fid = pn.fid
+        WHERE number in (:phones) WITH UR
+        """;
+
+    private static final String SELECT_NAME_BY_FIDS = """
+        SELECT
+            fid,
+            first,
+            middle,
+            name_1,
+            birth_dt,
+            file_since_dt as file_since_dt_name
+        FROM NAME 
+        WHERE fid in (:fids) WITH UR
+        """;
+
+    private final NamedParameterJdbcTemplate jdbcTemplate;
+
+    public PhoneRepository(@Qualifier("jdbcTemplateIndic") NamedParameterJdbcTemplate jdbcTemplate) {
+        this.jdbcTemplate = jdbcTemplate;
+    }
+
+
+    public List<FidsData> findFids(List<String> phones) {
+        MapSqlParameterSource params = new MapSqlParameterSource();
+        params.addValue("phones", phones);
+
+        return jdbcTemplate.query(SELCT_FIDS, params, (rs, rowNum) ->
+            new PhoneRepository.FidsData(rs.getInt("fid"), rs.getString("number"), rs.getObject("file_since_dt", LocalDate.class))
+        );
+    }
+
+    public List<Name> findNameByPhones(List<String> phones) {
+        MapSqlParameterSource params = new MapSqlParameterSource();
+        params.addValue("phones", phones);
+
+        return jdbcTemplate.query(SELECT_FIDS_WITH_NAME, params, (rs, rowNum) ->
+            Name.builder()
+                .fid(rs.getInt("FID"))
+                .number(rs.getString("NUMBER"))
+                .fileSinceDate(getLocalDate(rs, "FILE_SINCE_DT_NAME"))
+                .first(rs.getString("FIRST"))
+                .middle(rs.getString("MIDDLE"))
+                .last(rs.getString("NAME_1"))
+                .birthDate(getLocalDate(rs, "BIRTH_DT"))
+                .build());
+    }
+
+    public List<Name> findNameByFids(List<Integer> fids) {
+        MapSqlParameterSource params = new MapSqlParameterSource();
+        params.addValue("fids", fids);
+
+        return jdbcTemplate.query(SELECT_NAME_BY_FIDS, params, (rs, rowNum) ->
+            Name.builder()
+                .fid(rs.getInt("FID"))
+                .fileSinceDate(getLocalDate(rs, "FILE_SINCE_DT_NAME"))
+                .first(rs.getString("FIRST"))
+                .middle(rs.getString("MIDDLE"))
+                .last(rs.getString("NAME_1"))
+                .birthDate(getLocalDate(rs, "BIRTH_DT"))
+                .build());
+    }
+
+    private static LocalDate getLocalDate(ResultSet rs, String col) throws SQLException {
+        java.sql.Date d = rs.getDate(col);
+        return d != null ? d.toLocalDate() : null;
+    }
+
+    public List<FidsDataLong> findFidsLong(List<String> phones) {
+        MapSqlParameterSource params = new MapSqlParameterSource();
+        params.addValue("phones", phones);
+
+        return jdbcTemplate.query(SELCT_FIDS, params, (rs, rowNum) ->
+            new PhoneRepository.FidsDataLong(rs.getInt("fid"), Long.parseLong(rs.getString("number")), rs.getObject("file_since_dt", LocalDate.class))
+        );
+    }
+
+    @Getter
+    @Setter
+    @AllArgsConstructor
+    @NoArgsConstructor
+    public static class FidsData {
+        private Integer fid;
+        private String phone;
+        private LocalDate fileSinceDate;
+    }
+
+    @Getter
+    @Setter
+    @AllArgsConstructor
+    @NoArgsConstructor
+    public static class FidsDataWithName {
+        private Integer fid;
+        private String phone;
+        private LocalDate fileSinceDate;
+        private String firstName;
+        private String lastName;
+        private String middleName;
+        private LocalDate birthDate;
+        private LocalDate fileSinceDateName;
+    }
+
+    @Getter
+    @Setter
+    @AllArgsConstructor
+    @NoArgsConstructor
+    public static class FidsDataLong {
+        private Integer fid;
+        private Long phone;
+        private LocalDate fileSinceDate;
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/DbVersionValidationService.java

@@ -0,0 +1,62 @@
+package ru.nbch.credit_tracker.service;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.AppVersion;
+import ru.nbch.credit_tracker.mapper.signal.AppVersionMapper;
+
+@Service
+public class DbVersionValidationService implements InitializingBean {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    private final AppVersionMapper appVersionMapper;
+    private final CreditTrackerSettings settings;
+    private final String expectedDbVersion = "2.21";
+
+    @Autowired
+    public DbVersionValidationService(AppVersionMapper appVersionMapper, CreditTrackerSettings settings) {
+        this.appVersionMapper = appVersionMapper;
+        this.settings = settings;
+    }
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        String[] expectedVersions = expectedDbVersion.split("\\.");
+        String expectedMajor = expectedVersions[0];
+        String expectedMinor = expectedVersions[1];
+        if (StringUtils.isEmpty(expectedMajor) || StringUtils.isEmpty(expectedMinor)) {
+            log.error("expected database version is not specified in the settings.");
+            System.exit(1);
+        }
+
+        log.info("expected DB version: {}", expectedDbVersion);
+
+        AppVersion appVersion = appVersionMapper.getAppVersion();
+        String[] actualVersions = appVersion.getDbVersion().split("\\.");
+        String actualMajor = actualVersions[0];
+        String actualMinor = actualVersions[1];
+        if (StringUtils.isEmpty(actualMajor) || StringUtils.isEmpty(actualMinor)) {
+            log.error("couldn't load version from APP_VERSION database table.");
+            System.exit(1);
+        }
+
+        log.info("actual DB version: {}", appVersion.getDbVersion());
+
+        if (!expectedMajor.equals(actualMajor)) {
+            log.error("expected database version does not match actual database version");
+            System.exit(1);
+        }
+
+        if (Integer.parseInt(expectedMinor) < Integer.parseInt(actualMinor)) {
+            log.error("expected database version does not match actual database version");
+            System.exit(1);
+        }
+
+        log.info("Database version validation complete: {}", appVersion.getDbVersion());
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/MonitoringErrorProcesingService.java

@@ -1,14 +1,26 @@
 package ru.nbch.credit_tracker.service;
 
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Optional;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.aop.framework.AopContext;
 import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.mdc.SetAndClearMdc;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.PackagesRecordsMap;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.entities.external_response.monitoring_report_error.MonitoringReportErrors;
+import ru.nbch.credit_tracker.enums.RejectedStatus;
+import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
+import ru.nbch.credit_tracker.service.indic.PhoneService2;
 import ru.nbch.credit_tracker.service.signal.SignalRestService;
-import ru.nbch.credit_tracker.utils.CTUtil;
+import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
+import ru.nbch.credit_tracker.utils.CtFileUtils;
 
 @Slf4j
 @Service
@@ -16,38 +28,71 @@ import ru.nbch.credit_tracker.utils.CTUtil;
 public class MonitoringErrorProcesingService {
 
     private MonitoringErrorProcesingService self;
-
-    private final SignalService signalService;
+    private final UserPackageMapper userPackageMapper;
     private final SignalRestService signalRestService;
+    private final StaxXmlProcessor staxXmlProcessor;
+    private final PhoneService2 phoneService2;
+    private final Integer batchSize;
 
-    public MonitoringErrorProcesingService(SignalService signalService, SignalRestService signalRestService) {
-        this.signalService = signalService;
+    public MonitoringErrorProcesingService(UserPackageMapper userPackageMapper,
+                                           SignalRestService signalRestService,
+                                           StaxXmlProcessor staxXmlProcessor,
+                                           PhoneService2 phoneService2,
+                                           CreditTrackerSettings config) {
+        this.userPackageMapper = userPackageMapper;
         this.signalRestService = signalRestService;
+        this.staxXmlProcessor = staxXmlProcessor;
+        this.phoneService2 = phoneService2;
+        this.batchSize = config.getPipeline().subjectBatchSize();
     }
 
     public void findErrors(Long packageId) {
-        UserPackage userPackage = signalService.findPackage(packageId);
-        getCurrentProxy().processPackage(userPackage, packageId);
+        UserPackage userPackage = userPackageMapper.findPackage(packageId);
+        String packCode = userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName();
+        log.info("Проверка на ошибки при постановке на мониторинг. packCode={}, packageId={}", packCode, packageId);
+        String fileName = signalRestService.getErrorFileName(packageId, packCode);
+        if (StringUtils.isEmpty(fileName)) {
+            userPackageMapper.setRejectedStatus(packageId, RejectedStatus.OK.getKey());
+            return;
+        }
+
+        Optional<Path> filePath = signalRestService.loadErrorFile(fileName, userPackage);
+        if (filePath.isPresent()) {
+            try {
+                updateSubjectsErrors(filePath.get(), packageId);
+            } catch (Exception e) {
+                log.error("Ошибка при записи сообщений об ошибках в сформированных субъектах для мониторинга. Package_id = {}", packageId, e);
+            }
+            CtFileUtils.deleteQuietly(filePath.get());
+        }
+        userPackageMapper.setRejectedStatus(packageId, RejectedStatus.ERROR.getKey());
     }
 
-    @SetAndClearMdc
-    public void processPackage(UserPackage userPackage, Long packageId) {
-        if (userPackage != null) {
-            String packCode = userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName();
-            log.info("Проверка на ошибки при постановке на мониторинг. packCode={}, packageId={}", packCode, packageId);
-            String fileName = signalRestService.getErrorFileName(packageId, packCode);
-            if (StringUtils.isNotBlank(fileName)) {
-                String filePath = signalRestService.loadErrorFile(packageId, fileName);
-                if (StringUtils.isNotBlank(filePath)) {
-                    try {
-                        signalService.updateSubjectsErrors(filePath, packageId);
-                    } catch (Exception e) {
-                        log.error("Ошибка при записи сообщений об ошибках в сформированных субъектах для мониторинга. Package_id = {}", packageId, e);
+
+    private void updateSubjectsErrors(Path filePath, Long packageId) {
+        log.info("Start updating subjects errors. PackageId: {}", packageId);
+        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(filePath), 8 * 1024 * 1024)) {
+            staxXmlProcessor.read(in, MonitoringReportErrors.class,
+                request -> {
+                    if (!request.getPersons().isEmpty()) {
+                        List<PackagesRecordsMap> subjects = request.getPersons().stream().map(p -> {
+                            log.warn("Ошибка при формировании субъекта. package_id = {}, id = {}. Описание: {}", packageId, p.getUid(), p.getValue());
+                            return PackagesRecordsMap.builder()
+                                .id(p.getUid())
+                                .errorText(p.getValue())
+                                .build();
+                        }).toList();
+                        phoneService2.updateOnError(subjects);
+                        log.trace("Updated subject's batch. Total count: {}. PackageId: {}", request.getPersons().size(), packageId);
+                        request.getPersons().clear();
                     }
-                    CTUtil.removeFile(filePath);
-                }
-            }
+                },
+                request -> request.getPersons().isEmpty(),
+                batchSize, "/Errors/p");
+        } catch (IOException e) {
+            throw new RuntimeException("Error while parsing MonitoringReportErrors", e);
         }
+        log.info("Subject's errors were updated. PackageId: {}", packageId);
     }
 
     private MonitoringErrorProcesingService getCurrentProxy() {

src/main/java/ru/nbch/credit_tracker/service/MonitoringService.java

@@ -1,319 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.MDC;
-import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.Hit;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.entities.external_response.online_download.Inquiry;
-import ru.nbch.credit_tracker.entities.external_response.online_download.Person;
-import ru.nbch.credit_tracker.entities.external_response.online_download.Persons;
-import ru.nbch.credit_tracker.entities.external_response.online_report.Report;
-import ru.nbch.credit_tracker.entities.external_response.online_report.SgnlOnlineReport;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.MonitoringSignalResponse;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Signal;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Signals;
-import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Subject;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlWriter;
-import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
-import ru.nbch.credit_tracker.utils.CTUtil;
-
-import java.io.*;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.LocalDateTime;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-/**
- * Сервис периодического опроса официального сервиса сигналов на наличие новых отчетов по пакетам находящимся в статусе ON_MONITORING
- */
-@Slf4j
-@Service
-public class MonitoringService {
-
-    private final SignalRestService signalRestService;
-    private final SignalService signalService;
-    private final StaxXmlProcessor staxXmlProcessor;
-    private final CreditTrackerSettings config;
-
-    public MonitoringService(SignalRestService signalRestService,
-                             SignalService signalService,
-                             StaxXmlProcessor staxXmlProcessor,
-                             CreditTrackerSettings config) {
-        this.signalRestService = signalRestService;
-        this.signalService = signalService;
-        this.staxXmlProcessor = staxXmlProcessor;
-        this.config = config;
-    }
-
-    public void run() {
-        ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-"));
-        long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15;
-        scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES);
-    }
-
-    private void monitor() {
-        log.info("Start monitoring");
-        List<UserPackage> packages = signalService.findPackagesOnMonitoring();
-        for (UserPackage userPackage : packages) {
-            MDC.put("ru.nbch.credit_tracker.log.mdc.key", "[" + userPackage.getPackageName() +"]");
-            processPackage(userPackage);
-            MDC.remove("ru.nbch.credit_tracker.log.mdc.key");
-        }
-        log.info("Finish monitoring");
-    }
-
-    private void processPackage(UserPackage userPackage) {
-        log.info("Processing package");
-        if (!signalService.hasNonNullfids(userPackage.getId())) {
-            log.info("Package has no fids. Package will be skipped.");
-            return;
-        }
-        SgnlOnlineReport onlineReports = signalRestService.getReports(userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName());
-
-        if (onlineReports == null) {
-            log.info("No reports found for package");
-            return;
-        }
-
-        // Формируем список имен файлов, по которым сработали сигналы. Сюда входят только те имена, у которых парсится дата формирования отчета
-        List<String> fileNames = onlineReports.getOperationReport().getReports().getReports().stream().map(Report::getFilename)
-                .filter(fileName -> CTUtil.parseDateFromReportFileName(fileName) != null).toList();
-
-        if (fileNames.isEmpty()) {
-            log.info("No new report found for package");
-            return;
-        }
-
-        LocalDateTime lastDownloadedReportDate = CTUtil.parseDateFromReportFileName(userPackage.getLastDownloadedReport());
-
-        if (lastDownloadedReportDate != null) {
-            // Отбираем только те отчеты, дата которых позже даты последнего загруженного отчета
-            fileNames = fileNames.stream()
-                    .filter(fileName -> CTUtil.parseDateFromReportFileName(fileName).isAfter(lastDownloadedReportDate)) // отфильтровали только те имена, которые парсятся в дату, так что null тут не придет
-                    .sorted(Comparator.comparing(CTUtil::parseDateFromReportFileName)) // Сортируем по дате
-                    .toList();
-        } else {
-            fileNames = fileNames.stream()
-                    .sorted(Comparator.comparing(CTUtil::parseDateFromReportFileName)) // Сортируем по дате
-                    .toList();
-        }
-
-        if (StringUtils.isNotBlank(userPackage.getLastDownloadedReport())) {
-            log.info("Last downloaded report found for package. Last Downloaded Report Name: {}", userPackage.getLastDownloadedReport());
-        } else {
-            log.info("Last Downloaded Report Name is empty");
-        }
-
-        for (String newReport : fileNames) {
-            try {
-                String loadedReportPath = null;
-                if (StringUtils.isBlank(userPackage.getLastDownloadedReport())) {
-                    loadedReportPath = loadReport(userPackage.getId(), newReport); // по этому пакету еще не было загруженных отчетов, продолжаем работу с полученным
-                } else if (lastDownloadedReportDate != null) {
-                    LocalDateTime newReportDate = CTUtil.parseDateFromReportFileName(newReport);
-                    if (newReportDate.isAfter(lastDownloadedReportDate)) {
-                        loadedReportPath = loadReport(userPackage.getId(), newReport); // полученный отчет новее записанного в базу
-                    }
-                }
-                if (loadedReportPath == null) {
-                    log.info("Report was not loaded for package");
-                    continue;
-                }
-
-                String responsePath = generateResponse(userPackage, loadedReportPath);
-                String fileName = null;
-                if (responsePath != null) {
-                    fileName = createResponseFile(userPackage, responsePath);
-                }
-                if (fileName != null) {
-                    log.info("New report fileName: {}", fileName);
-                    updateSignalHits(userPackage, responsePath, fileName);
-                    signalService.updateLastDownloadedReport(userPackage.getId(), newReport);
-                } else {
-                    log.info("Report was not created for package");
-                }
-            } catch (Exception e) {
-                log.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    /**
-     * Формирует результирующий xml файл для конечного пользователя на основе файла из официального сервиса сигналов
-     * @return Путь к временному рещультирующему файлу
-     */
-    private String generateResponse(UserPackage userPackage, String loadedReportPath) {
-        AtomicBoolean generateResponse = new AtomicBoolean(false); // Выставляется в true, если хотя бы один субъект с мониторинга подошел под условие срабатывания сигнала. В этом случае отчет сохраняется
-
-        String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator;
-        String responsePath = folderPath + UUID.randomUUID() + "_response_output.xml";
-        CTUtil.createFolder(folderPath);
-
-        MonitoringSignalResponse response = new MonitoringSignalResponse();
-        response.setPackageName(userPackage.getPackageName()); // Имя пакета, переданного при установке
-
-        // Запись в результрующий файл
-        MonitoringSignalResponseStaxXmlWriter staxXmlWriter = new MonitoringSignalResponseStaxXmlWriter(config.getEncoding());
-        staxXmlWriter.writeWithStax(responsePath, response,
-                writer -> {
-                    try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(loadedReportPath)), 8 * 1024 * 1024)) {
-                        staxXmlProcessor.read(in, Persons.class,
-                                personsData -> { // как только считали пачку Persons сразу записали ее в результирующий файл
-                                    List<Subject> subjectsBatch = processPersons(personsData.getPersons());
-                                    if (!subjectsBatch.isEmpty()) {
-                                        generateResponse.set(true);
-                                    }
-                                    writer.accept(subjectsBatch);
-                                    personsData.getPersons().clear(); // чистим список для экономии памяти
-                                },
-                                pesons -> pesons.getPersons().isEmpty()
-                                ,
-                                10000,
-                                "/Report/Persons/Person");
-                    } catch (IOException e) {
-                        throw new RuntimeException("Error while parsing nbki downloaded report", e);
-                    }
-                });
-
-
-        CTUtil.removeFile(loadedReportPath);
-
-        if (!generateResponse.get()) { // ни один субъект не подошел под условие, удаляем временный отчет
-            CTUtil.removeFile(responsePath);
-            return null;
-        }
-
-        return responsePath;
-    }
-
-    private List<Subject> processPersons(List<Person> persons) {
-        List<Subject> subjects = new ArrayList<>();
-        for (Person person : persons) {
-            Signals signals = new Signals();
-            for (Inquiry inquiry : person.getInquiries()) {
-                // Отбираем только тех фл, у которых bus_category in (MFO, MKK, MFK) и есть сигналы с кодом 16
-                if (!CTUtil.checkBusCategory(inquiry.getBusCategory()) || inquiry.getSignals().stream().noneMatch(signal -> signal.getN().equals(16))) {
-                    continue;
-                }
-                Signal signal = new Signal();
-                signal.setSignalType(1); // Код типа сигнала по внутреннему справочнику. В текущей версии: 1 = сигнал 16 от MFO/MKK/MFK
-                signal.setInquiryDate(inquiry.getDate()); // Дата и время запроса КИ
-                signal.setInquiryPurpose(inquiry.getInqP()); // Цель запроса КИ
-                signals.getSignals().add(signal);
-            }
-
-            // Если есть хотя бы один заполненный сигнал, значит заполняем остальные данные
-            if (!signals.getSignals().isEmpty()) {
-                PhoneFidMap subjectData = signalService.findPhoneFidMapById(person.getUid());
-                if (subjectData != null) {
-                    Subject subject = new Subject();
-                    subject.setId(subjectData.getSubjectId()); // Уникальный идентификатор субъекта (равен Id из XML-пакета при установке мониторинга)
-                    subject.setClientId(subjectData.getPhone());
-                    subject.setFlagDebt5000(subjectData.getFlagDebt5000());
-                    subject.setFlag9012(subjectData.getFlag90days());
-                    subject.setSignals(signals);
-                    subjects.add(subject);
-                }
-            }
-        }
-        return subjects;
-    }
-
-    /**
-     * Архивирует результирующий xml файл для пользователя
-     * @return Путь к результрующему файлу. Пример <loadDir>/<member_code>/0001ZZ_MY_BATCH_TEST_0717_04_20250723_101929.zip
-     */
-    private String createResponseFile(UserPackage userPackage, String responsePath) {
-        try {
-            String fileName = CTUtil.generateResponseReportFileName(userPackage.getUserId(), userPackage.getPackageName());
-            Thread.sleep(1000); // Если отчеты формируются слишком быстро, нужно подождать секунду, тк в имени отчета LocalDateTime до секунд
-            String folderPath = config.getLoadDir() + File.separator + userPackage.getUserId().substring(0, 6) + File.separator;
-
-            File folder = new File(folderPath);
-            if (!folder.exists()) {
-                folder.mkdir();
-            }
-
-            try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(responsePath)), 8 * 1024 * 1024);
-                 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(folderPath + fileName + ".zip"));
-                 ZipOutputStream zos = new ZipOutputStream(bos)) {
-
-                ZipEntry zipEntry = new ZipEntry(fileName + ".xml");
-                zos.putNextEntry(zipEntry);
-                byte[] buffer = new byte[8 * 1024 * 1024];
-                int bytesRead;
-                while ((bytesRead = in.read(buffer)) > 0) {
-                    zos.write(buffer, 0, bytesRead);
-                }
-                zos.finish();
-                zos.closeEntry();
-            }
-            return fileName;
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-        } catch (Exception e) {
-            log.error("Ошибка при записи отчета пользователя в файл: ", e);
-        }
-
-        CTUtil.removeFile(responsePath);
-
-        return null;
-    }
-
-    private void updateSignalHits(UserPackage userPackage, String responsePath, String reportFileName) {
-        log.info("Updating signal hits for {}", userPackage.getPackageName());
-        LocalDateTime responseSentAt = LocalDateTime.now();
-        // Считываем результирующий файл пачками и обновляем таблицу hits
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(responsePath)), 8 * 1024 * 1024)) {
-            staxXmlProcessor.read(in, MonitoringSignalResponse.class,
-                    response -> { // как только считали пачку Subjects сразу обновлям таблицу hits
-                        List<Hit> hits = new ArrayList<>();
-                        response.getSubjects().getSubjects().forEach(subject -> {
-                            subject.getSignals().getSignals().forEach(signal -> {
-                                Hit hit = new Hit();
-                                hit.setPackageId(userPackage.getId());
-                                hit.setSubjectId(subject.getId());
-                                hit.setSignalType(1);
-                                hit.setResponseSentAt(responseSentAt);
-                                hit.setReportFileName(reportFileName);
-                                hits.add(hit);
-                            });
-                            if (!hits.isEmpty()) {
-                                signalService.insertHits(hits);
-                            }
-                        });
-                        response.getSubjects().getSubjects().clear(); // чистим список для экономии памяти
-                    },
-                    response -> response.getSubjects().getSubjects().isEmpty()
-                    ,
-                    10000,
-                    "/MonitoringSignalResponse/Subjects/Subject");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while updating hits", e);
-        }
-
-        CTUtil.removeFile(responsePath); // удалям результирующий xml файл с диска (Он уже заархивирован к этому моменту)
-
-        log.info("Signal hits updated for package {}", userPackage.getPackageName());
-    }
-
-    public String loadReport(Long packageId, String reportName) {
-        return signalRestService.downloadReport(packageId, reportName);
-    }
-
-}

src/main/java/ru/nbch/credit_tracker/service/SignalService.java

@@ -1,366 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.builder.PersonFactoryBuilder;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.Hit;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.entities.external_request.signals.Auth;
-import ru.nbch.credit_tracker.entities.external_request.signals.Person;
-import ru.nbch.credit_tracker.entities.external_request.signals.Signals;
-import ru.nbch.credit_tracker.entities.external_response.monitoring_report_error.MonitoringReportErrors;
-import ru.nbch.credit_tracker.entities.user_request.monitoring_request.MonitoringRequest;
-import ru.nbch.credit_tracker.enums.Status;
-import ru.nbch.credit_tracker.service.indic.FlagsService;
-import ru.nbch.credit_tracker.service.indic.PersonService;
-import ru.nbch.credit_tracker.service.indic.PhoneService;
-import ru.nbch.credit_tracker.service.signal.HitsService;
-import ru.nbch.credit_tracker.service.signal.PhoneFidMapService;
-import ru.nbch.credit_tracker.service.signal.UserPackageService;
-import ru.nbch.credit_tracker.service.xml.stax.SignalsStaxXmlWriter;
-import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
-import ru.nbch.credit_tracker.utils.CTUtil;
-
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.LocalDateTime;
-import java.util.*;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-
-@Slf4j
-@Service
-public class SignalService {
-
-    private final CreditTrackerSettings config;
-    private final HitsService hitsService;
-    private final UserPackageService userPackageService;
-    private final PhoneFidMapService phoneFidMapService;
-    private final PhoneService phoneService;
-    private final FlagsService flagsService;
-    private final PersonService personService;
-    private final StaxXmlProcessor staxXmlProcessor;
-
-    private final int batchSize = 10000;
-
-    public SignalService(CreditTrackerSettings config,
-                         HitsService hitsService,
-                         UserPackageService userPackageService,
-                         PhoneFidMapService phoneFidMapService,
-                         PhoneService phoneService,
-                         FlagsService flagsService,
-                         PersonService personService,
-                         StaxXmlProcessor staxXmlProcessor) {
-        this.config = config;
-        this.hitsService = hitsService;
-        this.userPackageService = userPackageService;
-        this.phoneFidMapService = phoneFidMapService;
-        this.phoneService = phoneService;
-        this.flagsService = flagsService;
-        this.personService = personService;
-        this.staxXmlProcessor = staxXmlProcessor;
-    }
-
-    public boolean isPackageNotUnique(MonitoringRequest request) {
-        return userPackageService.isPackageNotUnique(request.getUserId(), request.getPackageName());
-    }
-
-    public Long registerPackage(MonitoringRequest request) {
-        UserPackage userPackage = UserPackage.builder()
-                .userId(request.getUserId())
-                .packageName(request.getPackageName())
-                .createdAt(LocalDateTime.now())
-                .status(Status.PROCESSING.name())
-                .build();
-        return userPackageService.insertPackage(userPackage);
-    }
-
-    public void registerSubjects(String filePath, Long packageId) {
-        log.info("Start registering subjects. PackageId: {}", packageId);
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(filePath)), 8 * 1024 * 1024)) {
-            staxXmlProcessor.read(in, MonitoringRequest.class,
-                    request -> {
-                        if (!request.getSubjects().getSubject().isEmpty()) {
-                            phoneFidMapService.registerSubjects(request.getSubjects().getSubject(), packageId);
-                            log.trace("Registered subject's batch. Total count: {}. PackageId: {}", request.getSubjects().getSubject().size(), packageId);
-                            request.getSubjects().getSubject().clear();
-                        }
-                    },
-                    request -> request.getSubjects().getSubject().isEmpty()
-                    ,
-                    batchSize, "/MonitoringRequest/Subjects/Subject");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while parsing monitoring request", e);
-        }
-        log.info("Subjects were registered. PackageId: {}", packageId);
-    }
-
-    /**
-     * Обрабатывает все PhoneFidMap по packageId, выгружая их пачками и фильтруя по переданному фильтру
-     */
-    public UpdateResult updateSubjects(Long packageId, Predicate<PhoneFidMap> filter) {
-        boolean updated = false;
-        boolean noFidsFound = true;
-        int offset = 0;
-        List<PhoneFidMap> batch;
-        do {
-            batch = phoneFidMapService.findFullPhoneFidMap(packageId, offset, batchSize);
-
-            List<PhoneFidMap> filteredItems;
-            if (filter != null) {
-                filteredItems = batch.stream().filter(filter).toList();
-            } else {
-                filteredItems = batch;
-            }
-
-            if (!updated && !filteredItems.isEmpty()) {
-                updated = true;
-            }
-
-            if (!filteredItems.isEmpty()) {
-                boolean fidsFound = updateFlags(filteredItems, packageId);
-                if (noFidsFound) {
-                    noFidsFound = !fidsFound;
-                }
-            }
-
-            offset += batchSize;
-        } while (!batch.isEmpty());
-
-        return new UpdateResult(updated, noFidsFound);
-    }
-
-    /**
-     * Обрабатывает пачку PhoneFidMap: обновляет фиды и флаги просрочек
-     * @return fidsFound true - если хотя бы по одному номеру был найден фид, false - в противном случае
-     */
-    public boolean updateFlags(List<PhoneFidMap> subjects, Long packageId) {
-        log.info("Start calculating fids and flags for subjects batch. PackageId: {}", packageId);
-        Map<String, Integer> fidsMap = phoneService.findFidsPhoneData(subjects);
-        LocalDateTime calculatedAt = LocalDateTime.now();
-        boolean fidsFound = fidsMap.values().stream().anyMatch(Objects::nonNull);
-        Map<Integer, FlagsService.FlagsData> flagsDataMap = flagsService.findFlags(fidsMap.values().stream().filter(Objects::nonNull).toList());
-        List<PhoneFidMap> subjectToUpdate = new ArrayList<>();
-
-        subjects.forEach(subject -> {
-            Integer fid = fidsMap.get("9" + subject.getPhone());
-            if (fid != null) {
-                FlagsService.FlagsData flagsData = flagsDataMap.get(fid);
-                subject = PhoneFidMap.builder()
-                        .id(subject.getId())
-                        .packageId(packageId)
-                        .subjectId(subject.getSubjectId())
-                        .phone(subject.getPhone())
-                        .fid(fid)
-                        .flagDebt5000(flagsData.getFlagDebt5000())
-                        .flag90days(flagsData.getFlag90days())
-                        .calculatedAt(calculatedAt)
-                        .build();
-            } else {
-                subject = PhoneFidMap.builder()
-                        .id(subject.getId())
-                        .packageId(packageId)
-                        .subjectId(subject.getSubjectId())
-                        .phone(subject.getPhone())
-                        .calculatedAt(calculatedAt)
-                        .build();
-            }
-            subjectToUpdate.add(subject);
-        });
-
-        int updateBatchSize = batchSize / 9; // 9 параметров на одну запись для обновления. Максимальное количество параметров ~65000
-        for (int j = 0; j < subjectToUpdate.size(); j += updateBatchSize) {
-            int endIndex = Math.min(j + updateBatchSize, subjectToUpdate.size());
-            phoneFidMapService.updatePhoneFidMaps(subjectToUpdate.subList(j, endIndex));
-            log.trace("Updated fids and flags for subject's batch from: {} to: {}. Total count: {}. PackageId: {}", j, endIndex, subjectToUpdate.size(), packageId);
-        }
-
-        log.info("Finish calculating fids and flags for subjects. PackageId: {}. Batch size: {}", packageId, subjects.size());
-
-        return fidsFound;
-    }
-
-    public void rejectPackage(MonitoringRequest request) {
-        UserPackage userPackage = UserPackage.builder()
-                .userId(request.getUserId())
-                .packageName(request.getPackageName())
-                .createdAt(LocalDateTime.now())
-                .status(Status.REJECTED.name())
-                .build();
-        userPackageService.insertPackage(userPackage);
-    }
-
-    public void updatePackageStatus(Long packageId, Status status) {
-        userPackageService.updatePackageStatus(packageId, status);
-    }
-
-    public void updatePackageOnMonitoringSuccess(Long packageId, LocalDateTime monitoringAt) {
-        userPackageService.updatePackageOnMonitoringSuccess(packageId, monitoringAt);
-    }
-
-    /**
-     * Формирует файл запроса в официальный сервис сигналов, выгружая паспортные данные пачками и сразу записывая их в файл
-     *
-     * @return Путь к файлу запроса к официальному сервису сигналов на постановку на мониторинг
-     */
-    public String writeSignals(Long packageId) {
-        String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator;
-        String filePath = folderPath + UUID.randomUUID() + "_output.xml";
-        CTUtil.createFolder(folderPath);
-
-        Signals signals = buildSignals(packageId);
-
-        SignalsStaxXmlWriter signalsStaxXmlWriter = new SignalsStaxXmlWriter(config.getEncoding());
-
-        signalsStaxXmlWriter.writeWithStax(filePath,
-                signals,
-                writer -> buildPersons(packageId, writer));
-
-        return filePath;
-    }
-
-    private Signals buildSignals(Long packageId) {
-        UserPackage userPackage = userPackageService.findPackage(packageId);
-        Signals signals = new Signals();
-        signals.setOnline("2"); // Всегда=2 (только онлайн-сигналы)
-        signals.setProcessWithErrors("1"); // Всегда=1 (обработать пакет полностью, по обнаруженным ошибкам (при наличии) сформировать файл отказа, записи без ошибок установить на мониторинг)
-        signals.setId(userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName()); // <MEMBERCODE>_<PackageName>
-        Auth auth = new Auth();
-        auth.setUser(config.getSignalsuser());
-        signals.setAuth(auth);
-        return signals;
-    }
-
-    /**
-     * Обрабатывает все PhoneFidMap по packageId, выгружая их пачками и фильтруя по переданному фильтру.
-     * writer записывает пачку сразу в файл
-     */
-    private void buildPersons(Long packageId, Consumer<List<Person>> writer) {
-        int offset = 0;
-        List<PhoneFidMap> batch;
-        do {
-            batch = phoneFidMapService.findFullPhoneFidMap(packageId, offset, batchSize);
-            List<Person> persons = findPersons(batch);
-            if (!persons.isEmpty()) {
-                writer.accept(persons); // записывает пачку Persons в файл
-            }
-            offset += batchSize;
-        } while (!batch.isEmpty());
-
-    }
-
-    /**
-     * Формирует список Person для постановки на мониторинг в официальный сервис сигналов на основе списка PhoneFidMap
-     */
-    private List<Person> findPersons(List<PhoneFidMap> subjects) {
-        if (subjects.isEmpty()) {
-            return Collections.emptyList();
-        }
-        List<Person> persons = new ArrayList<>();
-        Map<Integer, PersonService.PersonData> personFidMap = personService.findPersonData(subjects.stream()
-                .map(PhoneFidMap::getFid)
-                .filter(Objects::nonNull).toList());
-        subjects.forEach(data -> {
-            if (data.getFid() != null) {
-                PersonService.PersonData personData = personFidMap.get(data.getFid());
-                Person person = PersonFactoryBuilder.builder()
-                        .personData(personData)
-                        .uid(data.getId())
-                        .build();
-                if (person != null) {
-                    persons.add(person);
-                }
-            }
-        });
-        return persons;
-    }
-
-    public void updateSubjectsErrors(String filePath, Long packageId) {
-        log.info("Start updating subjects errors. PackageId: {}", packageId);
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(filePath)), 8 * 1024 * 1024)) {
-            staxXmlProcessor.read(in, MonitoringReportErrors.class,
-                    request -> {
-                        if (!request.getPersons().isEmpty()) {
-                            List<PhoneFidMap> subjects = request.getPersons().stream().map(p -> {
-                                log.warn("Ошибка при формировании субъекта. package_id = {}, id = {}. Описание: {}", packageId, p.getUid(), p.getValue());
-                                return PhoneFidMap.builder()
-                                        .id(p.getUid())
-                                        .errorText(p.getValue())
-                                        .build();
-                            }).toList();
-                            phoneFidMapService.updatePhoneFidMapsErrors(subjects);
-                            log.trace("Updated subject's batch. Total count: {}. PackageId: {}", request.getPersons().size(), packageId);
-                            request.getPersons().clear();
-                        }
-                    },
-                    request -> request.getPersons().isEmpty()
-                    ,
-                    batchSize, "/Errors/p");
-        } catch (IOException e) {
-            throw new RuntimeException("Error while parsing MonitoringReportErrors", e);
-        }
-        log.info("Subject's errors were updated. PackageId: {}", packageId);
-    }
-
-    public List<UserPackage> findActivePackages(String userId) {
-        return userPackageService.findUserPackages(userId, Status.ON_MONITORING);
-    }
-
-    public List<UserPackage> findAllPackages(String userId) {
-        return userPackageService.findAllPackagesByUserId(userId);
-    }
-
-    public boolean isPackageExists(String username, String packageName) {
-        return userPackageService.isPackageExists(username, packageName, Status.ON_MONITORING);
-    }
-
-    public boolean isPackagesByStatusExist(String username, String packageName, Status status) {
-        return userPackageService.hasActivePackages(username, packageName, status);
-    }
-
-    public void deletePackage(String username, String packageName, LocalDateTime deactivatedAt) {
-        userPackageService.deletePackage(username, packageName, Status.DELETED, Status.ON_MONITORING, deactivatedAt);
-    }
-
-    public List<UserPackage> findPackagesOnMonitoring() {
-        return userPackageService.findAllPackagesByStatus(Status.ON_MONITORING);
-    }
-
-    public List<UserPackage> findPackagesOnProcessing() {
-        return userPackageService.findAllPackagesByStatus(Status.PROCESSING);
-    }
-
-    public void insertHits(List<Hit> hits) {
-        hitsService.insertHits(hits);
-    }
-
-    public void updateLastDownloadedReport(Long id, String fileName) {
-        userPackageService.updatePackageLastDownloadedReport(id, fileName);
-    }
-
-    public PhoneFidMap findPhoneFidMapById(Long id) {
-        return phoneFidMapService.findPhoneFidMapById(id);
-    }
-
-    public boolean hasNonNullfids(Long packageId) {
-        return phoneFidMapService.hasNonNullfids(packageId);
-    }
-
-    public UserPackage findPackage(Long packageId) {
-        return userPackageService.findPackage(packageId);
-    }
-
-    @Getter
-    @AllArgsConstructor
-    public static class UpdateResult {
-        private boolean updated;
-        private boolean noFidsFound;
-    }
-}

src/main/java/ru/nbch/credit_tracker/service/UpdateProcessingPackagesService.java

@@ -1,76 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.aop.framework.AopContext;
-import org.springframework.context.annotation.EnableAspectJAutoProxy;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.mdc.SetAndClearMdc;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.enums.Status;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-
-import java.util.List;
-
-/**
- * Сервис для завершения обработки пакетов находящихся в статусе PROCESSING
- * Запускается при старте приложения и отрабатывает 1 раз
- */
-@Slf4j
-@Service
-@EnableAspectJAutoProxy(exposeProxy = true)
-public class UpdateProcessingPackagesService {
-
-    private UpdateProcessingPackagesService self;
-
-    private final SignalService signalService;
-    private final SignalRestService signalRestService;
-    private final MonitoringErrorProcesingService monitoringErrorProcesingService;
-
-    public UpdateProcessingPackagesService(SignalService signalService,
-                                           SignalRestService signalRestService,
-                                           MonitoringErrorProcesingService monitoringErrorProcesingService) {
-        this.signalService = signalService;
-        this.signalRestService = signalRestService;
-        this.monitoringErrorProcesingService = monitoringErrorProcesingService;
-    }
-
-    public void run() {
-        log.info("Start updating processing packages");
-        List<UserPackage> packages = signalService.findPackagesOnProcessing();
-        for (UserPackage userPackage : packages) {
-            log.info("Updating flags for Package {}", userPackage.getPackageName());
-            getCurrentProxy().process(userPackage.getId(), userPackage.getPackageName());
-        }
-        log.info("Finish updating processing packages");
-    }
-
-    @SetAndClearMdc(argIndex = 1)
-    public void process(Long packageId, String packageName) {
-        try {
-            SignalService.UpdateResult updateResult = signalService.updateSubjects(packageId, null);
-            if (updateResult.isNoFidsFound()) {
-                log.info("No fids found for PackageId: {} . Package status ON_MONITORING. No registration in signal service.", packageId);
-                signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-                return;
-            }
-            if (signalRestService.setupMonitoring(packageId)) {
-                log.info("Package was successfully registered in signal service. PackageId: {}", packageId);
-                signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-                monitoringErrorProcesingService.findErrors(packageId);
-            } else {
-                log.info("Package registration was rejected by signal service. PackageId: {}", packageId);
-                signalService.updatePackageStatus(packageId, Status.ERROR);
-            }
-        } catch (Exception e) {
-            signalService.updatePackageStatus(packageId, Status.ERROR);
-            log.error("Ошибка при поиске фидов или расчете флагов просрочек. Package_id {}", packageId, e);
-        }
-    }
-
-    private UpdateProcessingPackagesService getCurrentProxy() {
-        if (self == null) {
-            self = (UpdateProcessingPackagesService) AopContext.currentProxy();
-        }
-        return self;
-    }
-}

src/main/java/ru/nbch/credit_tracker/service/UpdatingFlagsService.java

@@ -1,94 +0,0 @@
-package ru.nbch.credit_tracker.service;
-
-import lombok.extern.slf4j.Slf4j;
-import org.slf4j.MDC;
-import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.entities.dto.UserPackage;
-import ru.nbch.credit_tracker.enums.Status;
-import ru.nbch.credit_tracker.service.signal.SignalRestService;
-
-import java.time.LocalDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Predicate;
-
-/**
- * Сервис периодического обновления фидов и флагов просрочек по пакетам находящимся в статусе ON_MONITORING
- * По обновленным пакетам формируется новый запрос постановки на мониторинг в официальный сервис сигналов
- */
-@Slf4j
-@Service
-public class UpdatingFlagsService {
-
-    private final SignalService signalService;
-    private final SignalRestService signalRestService;
-    private final MonitoringErrorProcesingService monitoringErrorProcesingService;
-    private final Long daysToUpdateFlags;
-    private final Long updateFlagsInterval;
-
-    public UpdatingFlagsService(SignalService signalService,
-                                SignalRestService signalRestService,
-                                MonitoringErrorProcesingService monitoringErrorProcesingService,
-                                CreditTrackerSettings config) {
-        this.signalService = signalService;
-        this.signalRestService = signalRestService;
-        this.monitoringErrorProcesingService = monitoringErrorProcesingService;
-        this.daysToUpdateFlags = config.getDaysToUpdateFlags() != null ? config.getDaysToUpdateFlags() : 1;
-        this.updateFlagsInterval = config.getUpdateFlagsInterval() != null ? config.getUpdateFlagsInterval() : 1;
-    }
-
-    public void run() {
-        ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("updating-flags-"));
-        scheduler.scheduleAtFixedRate(this::update, 0, updateFlagsInterval, TimeUnit.DAYS);
-    }
-
-    private void update() {
-        log.info("Start updating flags");
-        List<UserPackage> packages = signalService.findPackagesOnMonitoring();
-        for (UserPackage userPackage : packages) {
-            log.info("Updating flags for Package {}", userPackage.getPackageName());
-            MDC.put("ru.nbch.credit_tracker.log.mdc.key", "[" + userPackage.getPackageName() +"]");
-            updateFlags(userPackage);
-            MDC.remove("ru.nbch.credit_tracker.log.mdc.key");
-        }
-        log.info("Finish updating flags");
-    }
-
-    private void updateFlags(UserPackage userPackage) {
-        Long packageId = userPackage.getId();
-        String packageName = userPackage.getPackageName();
-        try {
-            signalService.updatePackageStatus(packageId, Status.PROCESSING);
-            Predicate<PhoneFidMap> filter = subject -> subject.getCalculatedAt() != null && hasDaysPassed(subject.getCalculatedAt());
-            SignalService.UpdateResult updateResult = signalService.updateSubjects(packageId, filter);
-            if (updateResult.isUpdated()) {
-                if (signalRestService.setupMonitoring(packageId)) {
-                    LocalDateTime monitoringAt = userPackage.getMonitoringAt() != null ? userPackage.getMonitoringAt() : LocalDateTime.now();
-                    log.info("Flags were updated successfully. PackageId: {} PackageName: {} MonitoringAt: {}. Package status changed to ON_MONITORING", packageId, packageName, monitoringAt);
-                    signalService.updatePackageOnMonitoringSuccess(packageId, monitoringAt);
-                    monitoringErrorProcesingService.findErrors(packageId);
-                } else {
-                    log.info("Updated package registration was rejected by signal service. PackageId: {} PackageName: {}", packageId, packageName);
-                    signalService.updatePackageStatus(packageId, Status.ERROR);
-                }
-            } else {
-                log.info("No flags were updated for package id {}. Package status changed to ON_MONITORING", packageId);
-                signalService.updatePackageStatus(packageId, Status.ON_MONITORING);
-            }
-        } catch (Exception e) {
-            log.error("Ошибка при поиске фидов или перерасчете флагов просрочек. Package_id {}", packageId, e);
-        }
-    }
-
-    private boolean hasDaysPassed(LocalDateTime pastDateTime) {
-        LocalDateTime now = LocalDateTime.now();
-        long daysPassed = ChronoUnit.DAYS.between(pastDateTime, now);
-        return daysPassed >= daysToUpdateFlags;
-    }
-}

src/main/java/ru/nbch/credit_tracker/service/UserPackageService.java

@@ -0,0 +1,80 @@
+package ru.nbch.credit_tracker.service;
+
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.enums.Status;
+import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
+
+@Slf4j
+@Service
+public class UserPackageService {
+    private final UserPackageMapper userPackageMapper;
+
+    public UserPackageService(UserPackageMapper userPackageMapper) {
+        this.userPackageMapper = userPackageMapper;
+    }
+
+    public boolean isPackageNotUnique(String userId, String packageName) {
+        return userPackageMapper.isPackageNotUnique(userId.substring(0, 6), packageName);
+    }
+
+    public Long registerPackage(String userId, String packageName, List<Integer> signalTypes) {
+        signalTypes.sort(Comparator.naturalOrder());
+        String types = "1,2";
+        if (!signalTypes.isEmpty()) {
+            types = signalTypes.stream().map(Object::toString).distinct().collect(Collectors.joining(","));
+        }
+        UserPackage userPackage = UserPackage.builder()
+            .userId(userId)
+            .packageName(packageName)
+            .signalTypes(types)
+            .createdAt(LocalDateTime.now())
+            .status(Status.ACCEPTED.name())
+            .build();
+        return userPackageMapper.insertPackage(userPackage);
+    }
+
+    public void rejectPackage(String userId, String packageName, List<Integer> signalTypes) {
+        String types = "1,2";
+        if (!signalTypes.isEmpty()) {
+            types = signalTypes.stream().map(Object::toString).distinct().collect(Collectors.joining(","));
+        }
+        UserPackage userPackage = UserPackage.builder()
+            .userId(userId)
+            .packageName(packageName)
+            .signalTypes(types)
+            .createdAt(LocalDateTime.now())
+            .status(Status.REJECTED.name())
+            .build();
+        userPackageMapper.insertPackage(userPackage);
+    }
+
+    public List<UserPackage> findActivePackages(String userId) {
+        return userPackageMapper.findUserPackages(userId.substring(0, 6), Arrays.asList(Status.ON_MONITORING.name(), Status.NO_FID.name()));
+    }
+
+    public UserPackage findByPackageName(String packageName) {
+        return userPackageMapper.findByPackageName(packageName);
+    }
+
+    public void setFidChanged(List<Long> id) {
+        userPackageMapper.setFidChanged(id);
+    }
+    public List<UserPackage> findAllPackages(String userId) {
+        return userPackageMapper.findAllPackagesByUserId(userId.substring(0, 6));
+    }
+
+    public List<UserPackage> findByMemberCodeAndPackageName(String memberCode, String packageName) {
+        return userPackageMapper.findByMemberCodeAndPackageName(memberCode, packageName);
+    }
+
+    public void deletePackageById(Long id, LocalDateTime deactivatedAt) {
+        userPackageMapper.deletePackageById(id, Status.DELETED.name(), deactivatedAt);
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/ValidationService.java

@@ -1,5 +1,7 @@
-package ru.nbch.credit_tracker.utils;
+package ru.nbch.credit_tracker.service;
 
+import java.util.List;
+import java.util.regex.Pattern;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 import ru.nbch.credit_tracker.config.mdc.SetAndClearMdc;
@@ -8,18 +10,15 @@ import ru.nbch.credit_tracker.entities.user_request.monitoring_request.Subject;
 import ru.nbch.credit_tracker.enums.SignalError;
 import ru.nbch.credit_tracker.enums.XmlErrorFormat;
 import ru.nbch.credit_tracker.exceptions.SignalClientException;
-
-import java.util.regex.Pattern;
-
 import static ru.nbch.credit_tracker.utils.error.ExceptionUtils.clientExcp;
 
 @Slf4j
 @Service
 public class ValidationService {
 
-    private static final  String phoneRegex = "^\\d{9}$";
-    private static final  String packageNameRegex = "^[A-Z0-9_]{1,22}$";
-    private static final  String npIdRegex = "^\\S{1,36}$";
+    private static final String phoneRegex = "^\\d{9}$";
+    private static final String packageNameRegex = "^[A-Z0-9_]{1,22}$";
+    private static final String npIdRegex = "^\\S{1,36}$";
 
     @SetAndClearMdc
     public void validateMonitoringRequest(MonitoringRequest request) throws SignalClientException {
@@ -53,6 +52,14 @@ public class ValidationService {
             throw clientExcp(SignalError.ERROR_002, packageName, XmlErrorFormat.MonitoringError, "Subject");
         }
 
+        List<Integer> signalTypes = request.getSignalTypes();
+        if (signalTypes != null) {
+            boolean validValues = !signalTypes.isEmpty() && signalTypes.stream().allMatch(integer -> integer == 1 || integer == 2);
+            if (!validValues) {
+                throw clientExcp(SignalError.ERROR_027, packageName, XmlErrorFormat.MonitoringError);
+            }
+        }
+
         for (Subject subject : request.getSubjects().getSubject()) {
             if (!validateNPId(subject.getId())) {
                 throw clientExcp(SignalError.ERROR_010, packageName, XmlErrorFormat.MonitoringError, subject.getId());
@@ -64,21 +71,21 @@ public class ValidationService {
     }
 
     private static boolean validatePhone(String input) {
-        if(input == null) {
+        if (input == null) {
             return false;
         }
         return Pattern.matches(phoneRegex, input);
     }
 
     private static boolean validatePackageName(String input) {
-        if(input == null) {
+        if (input == null) {
             return false;
         }
         return Pattern.matches(packageNameRegex, input);
     }
 
     private static boolean validateNPId(String input) {
-        if(input == null) {
+        if (input == null) {
             return false;
         }
         return Pattern.matches(npIdRegex, input);

src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService.java

@@ -1,102 +1,16 @@
 package ru.nbch.credit_tracker.service.indic;
 
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.LocalDate;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.mapper.indic.FlagsMapper;
-
-import java.time.LocalDate;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
 
 @Service
 public class FlagsService {
-
-    private final FlagsMapper mapper;
-
-    public FlagsService(FlagsMapper mapper) {
-        this.mapper = mapper;
-    }
-
-    public Map<Integer, FlagsData> findFlags(List<Integer> fids) {
-        if (fids.isEmpty()) {
-            return new HashMap<>();
-        }
-        Map<Integer, FlagsData> result = new HashMap<>();
-        Map<Integer, Integer> flags5000 = calcFlag5000(fids);
-        Map<Integer, Integer> flags90DaysByPastDueArrear = calcFlag90DaysByPastDueArrear(fids);
-        Map<Integer, Integer> flags90DaysByRuPaymt = calcFlag90DaysByRuPaymt(fids);
-
-        fids.forEach(fid -> {
-            Integer flagDebt5000 = flags5000.get(fid);
-            if (flagDebt5000 == null) {
-                flagDebt5000 = 0;
-            }
-
-            Integer flag90days = flags90DaysByPastDueArrear.get(fid);
-            if (flag90days == null) {
-                flag90days = flags90DaysByRuPaymt.get(fid);
-                if (flag90days == null) {
-                    flag90days = 0;
-                }
-            }
-            result.put(fid, new FlagsData(flagDebt5000, flag90days));
-        });
-
-        return result;
-    }
-
-    /**
-     * По FID из  RU_PASTDUEARREAR выбираются все уникальные ACC_SERIAL_NUM с максимальной датой REPORTED_DT.
-     * Если по хотя бы одной такой записи AMT_PAST_DUE > 5000, то флаг устанавливается в 1.
-     * Если ни по одному счёту условие не выполняется или счетов вообще не найдено, флаг = 0.
-     */
-    private Map<Integer, Integer> calcFlag5000(List<Integer> fids) {
-        return mapper.findFlagsDebt5000(fids).stream()
-                .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag,
-                        (existing, replacement) -> {
-                            if (existing == 1) {
-                                return existing;
-                            } else {
-                                return replacement;
-                            }
-                        }));
-    }
-
-    /**
-     * Флаг = 1, если выполняется хотя бы одно из условий:
-     * В INDIC.RU_PASTDUEARREAR или RU_PAYMT у данного FID есть запись с DAYS_PAST_DUE ≥ 90 и REPORTED_DT >= текущая дата – 12 месяцев.
-     * Если ни по одной записи условия не выполняются или записей вообще не найдено, флаг = 0.
-     */
-    private Map<Integer, Integer> calcFlag90DaysByPastDueArrear(List<Integer> fids) {
-        LocalDate yearAgo = LocalDate.now().minusYears(1);
-        return mapper.findFlag90ByPastDueArrear(fids, yearAgo).stream()
-                .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag,
-                        (existing, replacement) -> {
-                            if (existing == 1) {
-                                return existing;
-                            } else {
-                                return replacement;
-                            }
-                        }));
-    }
-
-    private Map<Integer, Integer> calcFlag90DaysByRuPaymt(List<Integer> fids) {
-        LocalDate yearAgo = LocalDate.now().minusYears(1);
-        return mapper.findFlag90ByRuPaymt(fids, yearAgo).stream()
-                .collect(Collectors.toMap(FlagFidData::getFid, FlagFidData::getFlag,
-                        (existing, replacement) -> {
-                            if (existing == 1) {
-                                return existing;
-                            } else {
-                                return replacement;
-                            }
-                        }));
-    }
-
     @Getter
     @Setter
     @AllArgsConstructor
@@ -107,8 +21,23 @@ public class FlagsService {
 
     @Getter
     @Setter
-    public static class FlagFidData {
+    @AllArgsConstructor
+    @NoArgsConstructor
+    public static class RuPastDuarearData {
         private Integer fid;
-        private Integer flag;
+        private BigInteger accSerialNum;
+        private LocalDate reportedDate;
+        private BigDecimal amtPastDue;
+        private BigInteger daysPastDue;
+    }
+
+    @Getter
+    @Setter
+    @AllArgsConstructor
+    @NoArgsConstructor
+    public static class RuPaymtData {
+        private Integer fid;
+        private BigInteger daysPastDue;
+        private LocalDate reportedDate;
     }
 }

src/main/java/ru/nbch/credit_tracker/service/indic/FlagsService2.java

@@ -0,0 +1,48 @@
+package ru.nbch.credit_tracker.service.indic;
+
+import java.time.LocalDate;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.entities.indic.PaymentsFlag;
+import ru.nbch.credit_tracker.mapper.indic.IndicFlagsMapper;
+import ru.nbch.credit_tracker.mapper.signal.FlagDetailMapper;
+
+@Slf4j
+@Service
+public class FlagsService2 {
+    private final IndicFlagsMapper flagsMapper;
+    private final FlagDetailMapper flagDetailMapper;
+    private final ObjectMapper entityToJsonMapper;
+
+    public FlagsService2(IndicFlagsMapper flagsMapper,
+                         FlagDetailMapper flagDetailMapper,
+                         ObjectMapper entityToJsonMapper) {
+        this.flagsMapper = flagsMapper;
+        this.flagDetailMapper = flagDetailMapper;
+        this.entityToJsonMapper = entityToJsonMapper;
+    }
+
+    public Map<Integer, Integer> findFlagsDebt5000(List<Integer> fids) {
+        return flagsMapper.findFlagsDebt5000(fids).stream()
+            .collect(Collectors.toMap(PaymentsFlag::getFid, PaymentsFlag::getFlag));
+    }
+
+    public Map<Integer, Integer> findFlag90ByPastDueArrear(List<Integer> fids, LocalDate date) {
+        return flagsMapper.findFlag90ByPastDueArrear(fids, date).stream()
+            .collect(Collectors.toMap(PaymentsFlag::getFid, PaymentsFlag::getFlag));
+    }
+
+    public Map<Integer, Integer> findFlag90ByRuPaymt(List<Integer> fids, LocalDate date) {
+        return flagsMapper.findFlag90ByRuPaymt(fids, date).stream()
+            .collect(Collectors.toMap(PaymentsFlag::getFid, PaymentsFlag::getFlag));
+    }
+
+    public Map<Integer, Integer> findFlag90ByRuAllArrears(List<Integer> fids, LocalDate date) {
+        return flagsMapper.findFlag90ByRuAllArrears(fids, date).stream()
+            .collect(Collectors.toMap(PaymentsFlag::getFid, PaymentsFlag::getFlag));
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/indic/PersonService.java

@@ -1,20 +1,13 @@
 package ru.nbch.credit_tracker.service.indic;
 
+import java.time.LocalDate;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 import ru.nbch.credit_tracker.mapper.indic.PersonMapper;
 
-import java.time.LocalDate;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
 @Slf4j
 @Service
 public class PersonService {
@@ -25,132 +18,6 @@ public class PersonService {
         this.mapper = mapper;
     }
 
-    public Map<Integer, PersonData> findPersonData(List<Integer> fids) {
-        if (fids.isEmpty()) {
-            return new HashMap<>();
-        }
-        Map<Integer, PersonData> personDataMap = new HashMap<>();
-        Map<Integer, NameData> nameDataMap = findNameData(fids);
-        Map<Integer, PassportData> passortDataMap = findPassportData(fids);
-
-
-        fids.forEach(fid -> {
-            NameData person = nameDataMap.get(fid);
-            PassportData passort = passortDataMap.get(fid);
-            if (person != null && passort != null && requiredFieldPresent(person, passort)) {
-                personDataMap.put(fid, new PersonData(person, passort));
-            } else {
-                log.warn("Person does not have required fields. Fid {}", fid);
-            }
-        });
-
-        return personDataMap;
-    }
-
-
-    private Map<Integer, NameData> findNameData(List<Integer> fids) {
-        List<NameData> persons = mapper.findNameData(fids);
-        return persons.stream()
-                .collect(Collectors.toMap(NameData::getFid, nameData -> nameData,
-                        (existing, replacement) -> {
-                            LocalDate existingDate = existing.getFileSinceDate();
-                            LocalDate replacementDate = replacement.getFileSinceDate();
-                            if (existingDate != null && replacementDate != null) {
-                                if (existingDate.isAfter(replacementDate)) {
-                                    return existing;
-                                } else {
-                                    return replacement;
-                                }
-                            } else if (existingDate != null) {
-                                return existing;
-                            } else {
-                                return replacement;
-                            }
-                        }));
-    }
-
-    private Map<Integer, PassportData> findPassportData(List<Integer> fids) {
-        Map<Integer, PassportData> resultMap = new HashMap<>();
-
-        List<PassportData> passports = mapper.findPassportData(fids);
-
-        Map<Integer, List<PassportData>> passportsMap = passports.stream()
-                .collect(Collectors.groupingBy(PassportData::getFid));
-
-        fids.forEach(fid -> {
-            PassportData passport = findPassport(passportsMap.get(fid));
-            if (passport != null) {
-                resultMap.put(fid, passport);
-            }
-        });
-
-        return resultMap;
-    }
-
-    private PassportData findPassport(List<PassportData> passports) {
-        if (passports == null || passports.isEmpty()) {
-            return null;
-        }
-        List<PassportData> documents = passports.stream()
-                .filter(this::isDUL)
-                .toList(); // берем только ДУЛ
-
-        if (documents.isEmpty()) {
-            return null;
-        }
-
-        // Отдаем приоритет паспортам РФ
-        PassportData passport = documents.stream()
-                .filter(p -> "21".equals(p.getIdType2()) || "21".equals(p.getIdType()))
-                .sorted(
-                        Comparator.comparing(
-                                PassportData::getIssDate,
-                                Comparator.nullsLast(Comparator.reverseOrder())
-                        ).thenComparing(
-                                PassportData::getFileSinceDate,
-                                Comparator.nullsLast(Comparator.reverseOrder())
-                        )
-                ).findFirst().orElse(null);
-
-        if (passport != null) {
-            return passport;
-        }
-
-        // Если паспорт РФ не найден, смотри остальные ДУЛ'ы и берем первый
-        passport = documents.stream()
-                .sorted(
-                        Comparator.comparing(
-                                PassportData::getIssDate,
-                                Comparator.nullsLast(Comparator.reverseOrder())
-                        ).thenComparing(
-                                PassportData::getFileSinceDate,
-                                Comparator.nullsLast(Comparator.reverseOrder())
-                        )
-                ).findFirst().orElse(null);
-        return passport;
-    }
-
-    private boolean isDUL(PassportData p) {
-        Integer idType;
-        Double idType2;
-        try {
-            idType = p.getIdType() != null ? Integer.parseInt(p.getIdType()) : null;
-        } catch (NumberFormatException e) {
-            idType = null;
-        }
-        try {
-            idType2 = p.getIdType2() != null ? Double.valueOf(p.getIdType2()) : null;
-        } catch (NumberFormatException e) {
-            idType2 = null;
-        }
-        return (idType != null && idType <= 30 && idType != 12) || (idType2 != null && (idType2 >= 1 || idType2 <= 999));
-    }
-
-    private boolean requiredFieldPresent(NameData nameData, PassportData passportData) {
-        boolean isDocTypeNotNull = StringUtils.isNotBlank(passportData.idType) || StringUtils.isNotBlank(passportData.idType2);
-        return !StringUtils.isAllBlank(nameData.firstName, nameData.lastName, passportData.idNum) && nameData.birthDate != null && isDocTypeNotNull;
-    }
-
     @Getter
     @AllArgsConstructor
     public static class PersonData {

src/main/java/ru/nbch/credit_tracker/service/indic/PersonServiceV2.java

@@ -0,0 +1,136 @@
+package ru.nbch.credit_tracker.service.indic;
+
+import java.time.LocalDate;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.entities.indic.Id;
+import ru.nbch.credit_tracker.entities.indic.Name;
+import ru.nbch.credit_tracker.mapper.indic.PersonMapper;
+import ru.nbch.credit_tracker.model.SubjectProfile;
+import ru.nbch.credit_tracker.utils.CTUtil;
+
+@Slf4j
+@Service
+public class PersonServiceV2 {
+
+    private final PersonMapper mapper;
+    private final Comparator<Id> byDatesDesc =
+        Comparator.comparing(Id::getIssDate, Comparator.nullsLast(Comparator.naturalOrder()))
+            .thenComparing(Id::getFileSinceDate, Comparator.nullsLast(Comparator.naturalOrder()));
+
+    private final Comparator<Name> byDateThenIdNullSafe =
+        Comparator.comparing(Name::getFileSinceDatePn, Comparator.nullsFirst(Comparator.naturalOrder()))
+            .thenComparing(Name::getFileSinceDate, Comparator.nullsFirst(Comparator.naturalOrder()))
+            .thenComparing(Name::getFid);
+
+    public PersonServiceV2(PersonMapper mapper) {
+        this.mapper = mapper;
+    }
+
+    public Map<Integer, Id> filledPassportsData(List<Integer> fids) {
+        if (fids.isEmpty()) {
+            return new HashMap<>();
+        }
+        List<Id> passports = mapper.findPassportData(fids);
+        return groupPassportsData(passports);
+    }
+
+    public Map<Integer, Id> groupPassportsData(List<Id> passports) {
+        return passports.stream()
+            .filter(this::isDUL)
+            .filter(this::checkLength)
+            .collect(Collectors.groupingBy(
+                Id::getFid,
+                Collectors.collectingAndThen(
+                    Collectors.toList(),
+                    docs -> docs.stream()
+                        .filter(p -> "21".equals(p.getIdType2()) || "21".equals(p.getIdType()))
+                        .max(byDatesDesc)
+                        .or(() -> docs.stream().max(byDatesDesc))
+                        .orElse(null)
+                )
+            ));
+    }
+    /**
+     * Поиск данных по таблицам PHONE_NORM и NAME по списку номеров телефонов
+     *
+     * @param subjects
+     * @return - отображение NAME к номеру телефону, отфильтрованному по логике
+     * FID с максимальным значением FILE_SINCE_DT. Если FILE_SINCE_DT оказался одинаковый, берется максимальный FID
+     */
+    public Map<String, Name> findNameByNumber(List<SubjectProfile> subjects) {
+        List<String> phones = subjects.stream()
+            .map(subject -> CTUtil.normalizePhoneNumber(subject.getPhone()))
+            .toList();
+
+        List<Name> names = mapper.findNameData(phones);
+
+        return names.stream()
+            .filter(name ->
+                StringUtils.isNotEmpty(name.getFirst()) &&
+                    StringUtils.isNotEmpty(name.getLast()) &&
+                    name.getBirthDate() != null && !LocalDate.of(1, 1, 1).equals(name.getBirthDate())
+            )
+            .collect(Collectors.toMap(
+                Name::getNumber,
+                Function.identity(),
+                BinaryOperator.maxBy(byDateThenIdNullSafe)
+            ));
+    }
+
+    /**
+     * Поиск данных по таблицам PHONE_NORM и NAME по списку номеров телефонов
+     *
+     * @param subjects
+     * @return - отображение NAME к номеру телефону, отфильтрованному по логике
+     * FID с максимальным значением FILE_SINCE_DT. Если FILE_SINCE_DT оказался одинаковый, берется максимальный FID
+     */
+    public Map<String, Name> findNameByFids(List<SubjectProfile> subjects) {
+        List<Integer> fids = subjects.stream()
+            .map(SubjectProfile::getFid)
+            .toList();
+
+        List<Name> names = mapper.findNameDataByFids(fids);
+
+        return names.stream()
+            .filter(name ->
+                StringUtils.isNotEmpty(name.getFirst()) &&
+                    StringUtils.isNotEmpty(name.getLast()) &&
+                    name.getBirthDate() != null && !LocalDate.of(1, 1, 1).equals(name.getBirthDate())
+            )
+            .collect(Collectors.toMap(
+                Name::getNumber,
+                Function.identity(),
+                BinaryOperator.maxBy(byDateThenIdNullSafe)
+            ));
+    }
+
+    private boolean isDUL(Id p) {
+        Integer idType;
+        Double idType2;
+        try {
+            idType = p.getIdType() != null ? Integer.parseInt(p.getIdType()) : null;
+        } catch (NumberFormatException e) {
+            idType = null;
+        }
+        try {
+            idType2 = p.getIdType2() != null ? Double.valueOf(p.getIdType2()) : null;
+        } catch (NumberFormatException e) {
+            idType2 = null;
+        }
+        return (idType != null && idType <= 30 && idType != 12) ||
+                (idType2 != null && (idType2 >= 1 && idType2 <= 999));
+    }
+
+    private boolean checkLength(Id p) {
+        return (!"21".equals(p.getIdType2()) && !"21".equals(p.getIdType())) || CTUtil.normalizePassport(p).length() == 10;
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService.java

@@ -1,58 +0,0 @@
-package ru.nbch.credit_tracker.service.indic;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.springframework.stereotype.Service;
-import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
-import ru.nbch.credit_tracker.mapper.indic.PhoneMapper;
-
-import java.time.LocalDate;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-@Service
-public class PhoneService {
-
-    private final PhoneMapper mapper;
-
-    public PhoneService(PhoneMapper mapper) {
-        this.mapper = mapper;
-    }
-
-    /**
-     * Ищет фиды по номерам телефона
-     * @return Map: key - номер телефона; value - fid
-     */
-    public Map<String, Integer> findFidsPhoneData(List<PhoneFidMap> subjects) {
-        List<String> phones = subjects.stream().map(subject -> "9" + subject.getPhone()).toList();
-        return convertFidsDataToMap(mapper.findFids(phones));
-    }
-
-    private Map<String, Integer> convertFidsDataToMap(List<FidsData> fidsData) {
-        if (fidsData == null || fidsData.isEmpty()) {
-            return new HashMap<>();
-        }
-        return fidsData.stream()
-                .collect(Collectors.groupingBy(
-                        FidsData::getPhone,
-                        Collectors.collectingAndThen(
-                                Collectors.maxBy(
-                                        Comparator.comparing(FidsData::getFileSinceDate)
-                                                .thenComparing(FidsData::getFid)
-                                ),
-                                data -> data.map(FidsData::getFid).orElse(null)
-                        )
-                ));
-    }
-
-    @Getter
-    @Setter
-    public static class FidsData {
-        private Integer fid;
-        private String phone;
-        private LocalDate fileSinceDate;
-    }
-}

src/main/java/ru/nbch/credit_tracker/service/indic/PhoneService2.java

@@ -0,0 +1,93 @@
+package ru.nbch.credit_tracker.service.indic;
+
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.entities.app.PackagesRecordsMap;
+import ru.nbch.credit_tracker.entities.app.PhonesMap;
+import ru.nbch.credit_tracker.entities.indic.Name;
+import ru.nbch.credit_tracker.mapper.indic.PersonMapper;
+import ru.nbch.credit_tracker.mapper.signal.FlagDetailMapper;
+import ru.nbch.credit_tracker.mapper.signal.PackagesRecordsMapper;
+import ru.nbch.credit_tracker.repository.PhoneRepository;
+
+@Slf4j
+@Service
+public class PhoneService2 {
+
+    private final PhoneRepository phoneRepository;
+    private final PackagesRecordsMapper phoneFidMapMapper;
+    private final PersonMapper personMapper;
+    private final FlagDetailMapper flagDetailMapper;
+    private final ObjectMapper entityToJsonMapper;
+    private final Comparator<Name> byDateThenIdNullSafe =
+        Comparator.comparing(Name::getFileSinceDate, Comparator.nullsFirst(Comparator.naturalOrder()))
+            .thenComparing(Name::getFid);
+
+    public PhoneService2(PhoneRepository phoneRepository,
+                         PackagesRecordsMapper phoneFidMapMapper,
+                         PersonMapper personMapper,
+                         FlagDetailMapper flagDetailMapper,
+                         ObjectMapper entityToJsonMapper) {
+        this.phoneRepository = phoneRepository;
+        this.phoneFidMapMapper = phoneFidMapMapper;
+        this.personMapper = personMapper;
+        this.flagDetailMapper = flagDetailMapper;
+        this.entityToJsonMapper = entityToJsonMapper;
+    }
+
+    public List<Long> registerSubjects(List<PackagesRecordsMap> subjects) {
+        try {
+            String json = entityToJsonMapper.writeValueAsString(subjects.stream()
+                .map(phoneFidMap -> new HashMap<String, Object>() {{
+                    put("subject_id", phoneFidMap.getSubjectId());
+//                    put("fid", phoneFidMap.getFid());
+                    put("phone_l", phoneFidMap.getPhone_l());
+                    put("package_id", phoneFidMap.getPackageId());
+//                    put("calculated_at", phoneFidMap.getCalculatedAt());
+                }}).toList());
+            return phoneFidMapMapper.registerJson(json);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    public void updateFid(List<PhonesMap> subjects) {
+        try {
+            //регистрация в signals_phone_fid_map
+            String json = entityToJsonMapper.writeValueAsString(subjects.stream()
+                .map(phoneFidMap -> new HashMap<String, Object>() {{
+                    put("id", phoneFidMap.getId());
+                    put("fid", phoneFidMap.getFid());
+                    put("searched_at", phoneFidMap.getCalculatedAt());
+                }}).toList());
+            //fixme теперь апдейтится(insert) signals_phones_map
+//            phoneFidMapMapper.setFidJson(json);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void updateOnError(List<PackagesRecordsMap> subjects) {
+        try {
+            String json = entityToJsonMapper.writeValueAsString(subjects.stream()
+                .map(phoneFidMap -> new HashMap<String, Object>() {{
+                    put("id", phoneFidMap.getId());
+                    put("error_text", phoneFidMap.getErrorText());
+                }}).toList());
+            phoneFidMapMapper.setErrorJson(json);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public List<PackagesRecordsMap> pageByPackageId(Long packageId, Integer limit, Integer offset) {
+        return phoneFidMapMapper.findFullPhoneFidMap(packageId, limit, offset);
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/service/mode/common/CTMainApplication.java

@@ -0,0 +1,30 @@
+package ru.nbch.credit_tracker.service.mode.common;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
+
+@SpringBootApplication
+@ComponentScan(
+    value = {"ru.nbch.credit_tracker"},
+    excludeFilters = {
+        @ComponentScan.Filter(
+            type = FilterType.REGEX,
+            pattern = {
+                "ru\\.nbch\\.credit_tracker\\.service\\.mode\\.migration..*",
+                "ru\\.nbch\\.credit_tracker\\.service\\.mode\\.monitoring..*"
+            }
+        )
+    }
+)
+public class CTMainApplication {
+
+    public static void main(String[] args) {
+        ConfigurableApplicationContext ctx = SpringApplication.run(CTMainApplication.class, args);
+        StartupRestoreService updateProcessingPackagesService = ctx.getBean(StartupRestoreService.class);
+        updateProcessingPackagesService.run();
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/service/mode/common/StartupRestoreService.java

@@ -0,0 +1,103 @@
+package ru.nbch.credit_tracker.service.mode.common;
+
+import java.nio.file.Path;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.enums.Status;
+import ru.nbch.credit_tracker.mapper.signal.PackagesRecordsMapper;
+import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
+import ru.nbch.credit_tracker.service.MonitoringErrorProcesingService;
+import ru.nbch.credit_tracker.service.task.Pipeline;
+import ru.nbch.credit_tracker.service.task.PipelinePackageFabric;
+import ru.nbch.credit_tracker.service.task.PipelinePackageOrchestrator;
+import ru.nbch.credit_tracker.service.task.SetupParam;
+import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
+import ru.nbch.credit_tracker.service.task.impl.Saver;
+import ru.nbch.credit_tracker.utils.CTUtil;
+import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
+
+/**
+ * Сервис для завершения обработки пакетов находящихся в статусе PROCESSING
+ * Запускается при старте приложения и отрабатывает 1 раз
+ */
+@Slf4j
+@Service
+@EnableAspectJAutoProxy(exposeProxy = true)
+public class StartupRestoreService {
+
+    private final UserPackageMapper userPackageMapper;
+    private final PackagesRecordsMapper phoneFidMapMapper;
+    private final PipelinePackageFabric pipelinePackageManager;
+    private final PipelinePackageOrchestrator pipelineOrchestrator;
+    private final CreditTrackerSettings creditTrackerSettings;
+    private final MonitoringErrorProcesingService monitoringErrorProcesingService;
+    private final Saver saver;
+
+    public StartupRestoreService(UserPackageMapper userPackageMapper,
+                                 PackagesRecordsMapper phoneFidMapMapper,
+                                 PipelinePackageFabric pipelinePackageManager,
+                                 PipelinePackageOrchestrator pipelineOrchestrator,
+                                 CreditTrackerSettings creditTrackerSettings,
+                                 MonitoringErrorProcesingService monitoringErrorProcesingService,
+                                 Saver saver) {
+        this.userPackageMapper = userPackageMapper;
+        this.phoneFidMapMapper = phoneFidMapMapper;
+        this.pipelinePackageManager = pipelinePackageManager;
+        this.pipelineOrchestrator = pipelineOrchestrator;
+        this.creditTrackerSettings = creditTrackerSettings;
+        this.monitoringErrorProcesingService = monitoringErrorProcesingService;
+        this.saver = saver;
+    }
+
+    public void run() {
+        onMonitoringRestore();
+        acceptedRestore();
+        processingRestore();
+    }
+
+    private void onMonitoringRestore() {
+        List<UserPackage> packages = userPackageMapper.findOnMonitoringPackages();
+        packages.forEach(userPackage -> {
+            monitoringErrorProcesingService.findErrors(userPackage.getId());
+        });
+    }
+
+    private void acceptedRestore() {
+        List<UserPackage> packages = userPackageMapper.findAllPackagesByStatus(Status.ACCEPTED.name());
+        log.debug("Found {} packages in status ACCEPTED", packages.size());
+        for (UserPackage userPackage : packages) {
+            try {
+                Long count = phoneFidMapMapper.deleteByPackageId(userPackage.getId());
+                log.trace("Deleted {} rows", count);
+                Path xmlReqPath = CTUtil.workingPathToReqByPackage(creditTrackerSettings.getLoadDir(), userPackage);
+                log.debug("Try to read and build pipeline for packageId {}; req xml path: {}", userPackage.getId(), xmlReqPath);
+                Pipeline pipeline = pipelinePackageManager.build(new SetupParam(userPackage.getId(), xmlReqPath, PipelineMode.DEFAULT));
+                pipelineOrchestrator.startOrDelay(
+                    () -> saver.readAndSave(userPackage.getId(), xmlReqPath),
+                    pipeline, userPackage.getId()
+                );
+            } catch (Exception e) {
+                log.error("Error build default pipeline for packageId {}: {}", userPackage.getId(), ExceptionUtils.getStackTrace(e));
+            }
+        }
+    }
+
+    private void processingRestore() {
+        List<UserPackage> packages = userPackageMapper.findAllPackagesByStatus(Status.PROCESSING.name(), Status.PROCESSING_REPEATED.name());
+        log.debug("Found {} packages in status PROCESSING", packages.size());
+        for (UserPackage userPackage : packages) {
+            try {
+                log.debug("Build pipeline for packageId {}", userPackage.getId());
+                Pipeline pipeline = pipelinePackageManager.build(new SetupParam(userPackage.getId(), null, PipelineMode.DEFAULT));
+                pipelineOrchestrator.startOrDelay(pipeline, userPackage.getId());
+            } catch (Exception e) {
+                log.error("Error build restore pipeline for packageId {}: {}", userPackage.getId(), ExceptionUtils.getStackTrace(e));
+            }
+        }
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/service/mode/common/UpdatingFidAndPdService.java

@@ -0,0 +1,83 @@
+package ru.nbch.credit_tracker.service.mode.common;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
+import ru.nbch.credit_tracker.service.signal.PhonesMapService;
+import ru.nbch.credit_tracker.service.task.Pipeline;
+import ru.nbch.credit_tracker.service.task.PipelinePackageFabric;
+import ru.nbch.credit_tracker.service.task.PipelinePackageOrchestrator;
+import ru.nbch.credit_tracker.service.task.PipelineRecalcPDOrchestrator;
+import ru.nbch.credit_tracker.service.task.SetupParam;
+import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
+
+@Slf4j
+@Service
+@EnableScheduling
+public class UpdatingFidAndPdService {
+    private final PipelinePackageFabric pipelinePackageManager;
+    private final PipelineRecalcPDOrchestrator pipelineRecalcPDOrchestrator;
+    private final PipelinePackageOrchestrator pipelinePackageOrchestrator;
+    private final UserPackageMapper userPackageMapper;
+    private final PhonesMapService phonesMapService;
+
+    public UpdatingFidAndPdService(CreditTrackerSettings config,
+                                   PipelinePackageFabric pipelinePackageManager,
+                                   PipelineRecalcPDOrchestrator pipelineRecalcPDOrchestrator,
+                                   PipelinePackageOrchestrator pipelinePackageOrchestrator,
+                                   UserPackageMapper userPackageMapper,
+                                   PhonesMapService phonesMapService) {
+        this.pipelinePackageManager = pipelinePackageManager;
+        this.pipelineRecalcPDOrchestrator = pipelineRecalcPDOrchestrator;
+        this.pipelinePackageOrchestrator = pipelinePackageOrchestrator;
+        this.userPackageMapper = userPackageMapper;
+        this.phonesMapService = phonesMapService;
+    }
+
+    @Scheduled(cron = "${ct.update-fid-cron-expression}")
+    public void update() {
+        if (!pipelineRecalcPDOrchestrator.checkPermits()) {
+            log.info("The previous task has not yet been completed.");
+            return;
+        }
+        log.info("Delete unused phones");
+        phonesMapService.deleteUnusedPhones();
+        log.info("Start updating fids and PD");
+        try {
+            Pipeline pipeline = pipelinePackageManager.buildRecalculationPD(new SetupParam(null, null, PipelineMode.RECALC_PERSONAL_DATA));
+            CompletableFuture<?> future = pipelineRecalcPDOrchestrator.startOrReject(pipeline);
+            future.whenComplete((o, throwable) -> {
+                try {
+                    if (throwable != null) {
+                        log.error(throwable.getMessage(), throwable);
+                        return;
+                    }
+
+                    List<UserPackage> packagesRepeated = userPackageMapper.findForRepeatedPackages();
+                    packagesRepeated.forEach(userPackage -> {
+                        Long packageId = userPackage.getId();
+                        SetupParam setupParam = new SetupParam(packageId, null, PipelineMode.RESEND);
+                        try {
+                            Pipeline makeXmlByCache = pipelinePackageManager.build(setupParam);
+                            pipelinePackageOrchestrator.startOrDelay(makeXmlByCache, packageId);
+                        } catch (Exception e) {
+                            log.error("Failed to build pipeline", e);
+                        }
+                    });
+                } finally {
+                    log.info("pipeline finished");
+                    pipeline.close();
+                    pipelineRecalcPDOrchestrator.release();
+                }
+            });
+        } catch (Exception e) {
+            log.error("Failed to build pipeline", e);
+        }
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/mode/common/UpdatingFlagsService.java

@@ -0,0 +1,96 @@
+package ru.nbch.credit_tracker.service.mode.common;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.PhonesMap;
+import ru.nbch.credit_tracker.mapper.signal.PhonesMapMapper;
+import ru.nbch.credit_tracker.service.task.impl.FlagsEnricher;
+
+/**
+ * Сервис периодического обновления фидов и флагов просрочек по пакетам находящимся в статусе ON_MONITORING
+ * По обновленным пакетам формируется новый запрос постановки на мониторинг в официальный сервис сигналов
+ */
+@Slf4j
+@Service
+@EnableScheduling
+public class UpdatingFlagsService {
+    private final Logger log = LoggerFactory.getLogger(this.getClass());
+    private final static int BATCH_SIZE = 1000;
+    private final PhonesMapMapper phonesMapMapper;
+    private final FlagsEnricher flagsEnricherV2;
+    private final Integer updateDaysInterval;
+    private final AtomicBoolean insertIsRunning = new AtomicBoolean(false);
+    private final AtomicBoolean updateIsRunning = new AtomicBoolean(false);
+
+    public UpdatingFlagsService(CreditTrackerSettings settings,
+                                PhonesMapMapper phonesMapMapper,
+                                FlagsEnricher flagsEnricherV2) {
+        this.phonesMapMapper = phonesMapMapper;
+        this.flagsEnricherV2 = flagsEnricherV2;
+        this.updateDaysInterval = settings.getDaysToUpdateFlags();
+    }
+
+    @Scheduled(cron = "${ct.update-flags-cron-expression}")
+    private void updateFlags() {
+        if (!updateIsRunning.compareAndSet(false, true)) {
+            log.error("update flags processor already running... skipping scheduled update");
+            return;
+        }
+        try {
+            log.info("start update flags processor");
+            long lastId = 0L;
+            int batchNumber = 1;
+            List<PhonesMap> phonesMaps;
+            while (true) {
+                phonesMaps = phonesMapMapper.findEligibleForFlagsUpdate(lastId, updateDaysInterval, BATCH_SIZE);
+                log.trace("update batch {}, signals_phones_map records loaded: {}", batchNumber, phonesMaps.size());
+                if (phonesMaps.isEmpty()) {
+                    break;
+                }
+
+                flagsEnricherV2.accept(phonesMaps);
+                lastId = phonesMaps.getLast().getId();
+                log.trace("update batch {}, lastId {}", batchNumber++, lastId);
+            }
+            log.info("end update flags processor");
+        } finally {
+            updateIsRunning.set(false);
+        }
+    }
+
+    @Scheduled(cron = "${ct.new-flags-cron-expression}")
+    private void newFlags() {
+        if (!insertIsRunning.compareAndSet(false, true)) {
+            log.warn("new flags processor already running... skipping scheduled insert");
+            return;
+        }
+        try {
+            log.info("start processing new flags");
+            long lastId = 0L;
+            int batchNumber = 1;
+            List<PhonesMap> phonesMaps;
+            while (true) {
+                phonesMaps = phonesMapMapper.findEligibleForFlagsInsert(lastId, BATCH_SIZE);
+                log.trace("new batch {}, signals_phones_map records loaded: {}", batchNumber, phonesMaps.size());
+                if (phonesMaps.isEmpty()) {
+                    break;
+                }
+
+                flagsEnricherV2.accept(phonesMaps);
+                lastId = phonesMaps.getLast().getId();
+                log.trace("new batch {}, lastId {}", batchNumber++, lastId);
+            }
+            log.info("end new flags processor");
+        } finally {
+            insertIsRunning.set(false);
+        }
+
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/mode/migration/CTMigrationApplication.java

@@ -0,0 +1,54 @@
+package ru.nbch.credit_tracker.service.mode.migration;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
+import ru.nbch.credit_tracker.component.MonitoringRequestValidator;
+import ru.nbch.credit_tracker.config.datasource.ScoringDataSourceConfig;
+import ru.nbch.credit_tracker.config.rest.RestConfig;
+import ru.nbch.credit_tracker.config.xml.XmlConfig;
+import ru.nbch.credit_tracker.controller.SignalControllerV3;
+import ru.nbch.credit_tracker.facade.SignalFacadeV3;
+import ru.nbch.credit_tracker.service.MonitoringErrorProcesingService;
+import ru.nbch.credit_tracker.service.mode.common.CTMainApplication;
+import ru.nbch.credit_tracker.service.mode.common.StartupRestoreService;
+import ru.nbch.credit_tracker.service.mode.common.UpdatingFidAndPdService;
+import ru.nbch.credit_tracker.service.mode.common.UpdatingFlagsService;
+import ru.nbch.credit_tracker.service.mode.monitoring.CTMonitoringApplication;
+import ru.nbch.credit_tracker.service.mode.monitoring.MonitoringService;
+import ru.nbch.credit_tracker.service.scoring.AuthorizationService;
+import ru.nbch.credit_tracker.service.signal.SignalRestService;
+import ru.nbch.credit_tracker.service.task.PipelinePackageFabric;
+import ru.nbch.credit_tracker.service.task.PipelinePackageOrchestrator;
+import ru.nbch.credit_tracker.service.task.impl.FlagsEnricher;
+import ru.nbch.credit_tracker.service.task.impl.Saver;
+
+@SpringBootApplication
+@ComponentScan(
+    basePackages = {"ru.nbch.credit_tracker"},
+    excludeFilters = {
+        @ComponentScan.Filter(
+            type = FilterType.ASSIGNABLE_TYPE,
+            classes = {CTMonitoringApplication.class, CTMainApplication.class,
+                ScoringDataSourceConfig.class,StartupRestoreService.class, UpdatingFidAndPdService.class,
+                UpdatingFlagsService.class, MonitoringService.class, AuthorizationService.class,
+                SignalFacadeV3.class, SignalControllerV3.class, FlagsEnricher.class, PipelinePackageFabric.class,
+                RestConfig.class, SignalRestService.class, MonitoringErrorProcesingService.class, Saver.class,
+                PipelinePackageOrchestrator.class, XmlConfig.class, MonitoringRequestValidator.class
+            }
+        )
+    }
+)
+public class CTMigrationApplication {
+    public static void main(String[] args) {
+        SpringApplication application = new SpringApplication(CTMigrationApplication.class);
+        application.setWebApplicationType(WebApplicationType.NONE);
+        ConfigurableApplicationContext ctx = application.run(args);
+        MigrationService migrationService = ctx.getBean(MigrationService.class);
+        migrationService.update();
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/service/mode/migration/MigrationService.java

@@ -0,0 +1,61 @@
+package ru.nbch.credit_tracker.service.mode.migration;
+
+import java.util.concurrent.CompletableFuture;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.SpringApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.service.task.Pipeline;
+import ru.nbch.credit_tracker.service.task.PipelineMigrationPackageFabric;
+import ru.nbch.credit_tracker.service.task.PipelineRecalcPDOrchestrator;
+import ru.nbch.credit_tracker.service.task.SetupParam;
+import ru.nbch.credit_tracker.service.task.impl.PipelineMode;
+
+@Slf4j
+@Service
+@EnableScheduling
+public class MigrationService {
+    private final PipelineMigrationPackageFabric pipelinePackageManager;
+    private final PipelineRecalcPDOrchestrator pipelineRecalcPDOrchestrator;
+    private final ConfigurableApplicationContext context;
+
+    public MigrationService(PipelineMigrationPackageFabric pipelinePackageManager,
+                            PipelineRecalcPDOrchestrator pipelineRecalcPDOrchestrator,
+                            ConfigurableApplicationContext context) {
+        this.pipelinePackageManager = pipelinePackageManager;
+        this.pipelineRecalcPDOrchestrator = pipelineRecalcPDOrchestrator;
+        this.context = context;
+    }
+
+    public void update() {
+        log.info("Start updating fids and PD");
+        try {
+            Pipeline pipeline = pipelinePackageManager.buildRecalculationPDMigration(new SetupParam(null, null, PipelineMode.RECALC_PERSONAL_DATA));
+            CompletableFuture<?> future = pipelineRecalcPDOrchestrator.startOrReject(pipeline);
+            future.whenComplete((o, throwable) -> {
+                try {
+                    if (throwable != null) {
+                        log.error(throwable.getMessage(), throwable);
+                    }
+                } finally {
+                    log.info("pipeline finished");
+                    pipeline.close();
+                    pipelineRecalcPDOrchestrator.release();
+                    new Thread(this::shutdown, "migration-app-exit").start();
+                }
+            });
+        } catch (Exception e) {
+            log.error("Failed to build pipeline", e);
+        }
+    }
+
+    private void shutdown() {
+        int exitCode = SpringApplication.exit(context, () -> {
+            // no errors
+            return 0;
+        });
+        System.exit(exitCode);
+    }
+
+}

src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/CTMonitoringApplication.java

@@ -0,0 +1,31 @@
+package ru.nbch.credit_tracker.service.mode.monitoring;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
+
+@SpringBootApplication
+@ComponentScan(
+    value = {"ru.nbch.credit_tracker"},
+    excludeFilters = {
+        @ComponentScan.Filter(
+            type = FilterType.REGEX,
+            pattern = {
+                "ru\\.nbch\\.credit_tracker\\.service\\.mode\\.common..*",
+                "ru\\.nbch\\.credit_tracker\\.service\\.mode\\.migration..*"
+            }
+        )
+    }
+)
+public class CTMonitoringApplication {
+    public static void main(String[] args) {
+        SpringApplication application = new SpringApplication(CTMonitoringApplication.class);
+        application.setWebApplicationType(WebApplicationType.NONE);
+        ConfigurableApplicationContext ctx = application.run(args);
+        MonitoringService monitoringService = ctx.getBean(MonitoringService.class);
+        monitoringService.run();
+    }
+}

src/main/java/ru/nbch/credit_tracker/service/mode/monitoring/MonitoringService.java

@@ -0,0 +1,330 @@
+package ru.nbch.credit_tracker.service.mode.monitoring;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.MDC;
+import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
+import org.springframework.stereotype.Service;
+import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
+import ru.nbch.credit_tracker.entities.app.Hit;
+import ru.nbch.credit_tracker.entities.app.PackagesRecordsMap;
+import ru.nbch.credit_tracker.entities.app.PhonesMap;
+import ru.nbch.credit_tracker.entities.app.UserPackage;
+import ru.nbch.credit_tracker.entities.external_response.online_download.Inquiry;
+import ru.nbch.credit_tracker.entities.external_response.online_download.Person;
+import ru.nbch.credit_tracker.entities.external_response.online_download.Persons;
+import ru.nbch.credit_tracker.entities.external_response.online_report.Report;
+import ru.nbch.credit_tracker.entities.external_response.online_report.SgnlOnlineReport;
+import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.MonitoringSignalResponse;
+import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Signal;
+import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Signals;
+import ru.nbch.credit_tracker.entities.user_response.monitoring_signal_response.Subject;
+import ru.nbch.credit_tracker.mapper.signal.FlagDetailMapper;
+import ru.nbch.credit_tracker.mapper.signal.UserPackageMapper;
+import ru.nbch.credit_tracker.service.signal.HitsService;
+import ru.nbch.credit_tracker.service.signal.PackageRecordsService;
+import ru.nbch.credit_tracker.service.signal.SignalRestService;
+import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlWriter;
+import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
+import ru.nbch.credit_tracker.utils.CTUtil;
+import ru.nbch.credit_tracker.utils.CtFileUtils;
+import ru.nbch.credit_tracker.utils.ZipUtils;
+import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
+
+/**
+ * Сервис периодического опроса официального сервиса сигналов на наличие новых отчетов по пакетам находящимся в статусе ON_MONITORING
+ */
+@Slf4j
+@Service
+public class MonitoringService {
+
+    private final SignalRestService signalRestService;
+    private final FlagDetailMapper flagDetailMapper;
+    private final UserPackageMapper userPackageMapper;
+    private final StaxXmlProcessor staxXmlProcessor;
+    private final HitsService hitsService;
+    private final PackageRecordsService packageRecordsService;
+    private final CreditTrackerSettings config;
+
+    public MonitoringService(SignalRestService signalRestService,
+                             FlagDetailMapper flagDetailMapper,
+                             UserPackageMapper userPackageMapper,
+                             StaxXmlProcessor staxXmlProcessor,
+                             HitsService hitsService,
+                             PackageRecordsService packageRecordsService,
+                             CreditTrackerSettings config) {
+        this.signalRestService = signalRestService;
+        this.flagDetailMapper = flagDetailMapper;
+        this.userPackageMapper = userPackageMapper;
+        this.staxXmlProcessor = staxXmlProcessor;
+        this.hitsService = hitsService;
+        this.packageRecordsService = packageRecordsService;
+        this.config = config;
+    }
+
+    public void run() {
+        log.debug("Running scheduler for monitoring by membercode: {}", config.getMonitoringMembercodes());
+        ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("monitoring-"));
+        long period = config.getMonitoringInterval() != null ? config.getMonitoringInterval() : 15;
+        scheduler.scheduleAtFixedRate(this::monitor, 0, period, TimeUnit.MINUTES);
+    }
+
+    private void monitor() {
+        try {
+            log.info("Start monitoring");
+            List<UserPackage> packages = userPackageMapper.findOnMonitoringByMemberCode(config.getMonitoringMembercodes());
+            for (UserPackage userPackage : packages) {
+                MDC.put("ru.nbch.credit_tracker.log.mdc.key", "[" + userPackage.getPackageName() + "]");
+                processPackage(userPackage);
+                MDC.remove("ru.nbch.credit_tracker.log.mdc.key");
+            }
+            log.info("Finish monitoring");
+        } catch (Exception e) {
+            log.error("{}", ExceptionUtils.getStackTrace(e));
+        }
+    }
+
+    private void processPackage(UserPackage userPackage) {
+        log.info("Processing package");
+//        if (!phoneFidMapMapper.hasNonNullFids(userPackage.getId())) {
+//            log.info("Package has no fids. Package will be skipped.");
+//            return;
+//        }
+        SgnlOnlineReport onlineReports = signalRestService.getReports(userPackage.getUserId().substring(0, 6) + "_" + userPackage.getPackageName());
+
+        if (onlineReports == null) {
+            log.info("No reports found for package");
+            return;
+        }
+
+        // Формируем список имен файлов, по которым сработали сигналы. Сюда входят только те имена, у которых парсится дата формирования отчета
+        List<String> fileNames = onlineReports.getOperationReport().getReports().getReports().stream()
+            .map(Report::getFilename)
+            .filter(fileName -> CTUtil.parseDateFromReportFileName(fileName) != null).toList();
+
+        if (fileNames.isEmpty()) {
+            log.info("No new report found for package");
+            return;
+        }
+
+        LocalDateTime lastDownloadedReportDate = CTUtil.parseDateFromReportFileName(userPackage.getLastDownloadedReport());
+
+        if (lastDownloadedReportDate != null) {
+            // Отбираем только те отчеты, дата которых позже даты последнего загруженного отчета
+            fileNames = fileNames.stream()
+                .filter(fileName -> CTUtil.parseDateFromReportFileName(fileName).isAfter(lastDownloadedReportDate)) // отфильтровали только те имена, которые парсятся в дату, так что null тут не придет
+                .sorted(Comparator.comparing(CTUtil::parseDateFromReportFileName)) // Сортируем по дате
+                .toList();
+        } else {
+            fileNames = fileNames.stream()
+                .sorted(Comparator.comparing(CTUtil::parseDateFromReportFileName)) // Сортируем по дате
+                .toList();
+        }
+
+        if (StringUtils.isNotBlank(userPackage.getLastDownloadedReport())) {
+            log.info("Last downloaded report found for package. Last Downloaded Report Name: {}", userPackage.getLastDownloadedReport());
+        } else {
+            log.info("Last Downloaded Report Name is empty");
+        }
+
+        for (String newReport : fileNames) {
+            try {
+                Optional<Path> loadedReportPath = Optional.empty();
+                if (StringUtils.isBlank(userPackage.getLastDownloadedReport())) {
+                    // по этому пакету еще не было загруженных отчетов, продолжаем работу с полученным
+                    loadedReportPath = signalRestService.downloadReport(userPackage, newReport);
+                } else if (lastDownloadedReportDate != null) {
+                    LocalDateTime newReportDate = CTUtil.parseDateFromReportFileName(newReport);
+                    if (newReportDate.isAfter(lastDownloadedReportDate)) {
+                        // полученный отчет новее записанного в базу
+                        loadedReportPath = signalRestService.downloadReport(userPackage, newReport);
+                    }
+                }
+                if (loadedReportPath.isEmpty()) {
+                    log.info("Report was not loaded for package");
+                    continue;
+                }
+
+                Path responsePath = generateResponse(userPackage, newReport, loadedReportPath.get());
+                if (responsePath != null) {
+                    Path zipFile = ZipUtils.zipFile(responsePath);
+                    log.info("New report file by path: {}", zipFile);
+                    updateSignalHits(userPackage, responsePath, zipFile);
+                } else {
+                    log.info("Report was not created for package");
+                }
+                userPackageMapper.updatePackageLastDownloadedReport(userPackage.getId(), newReport);
+            } catch (Exception e) {
+                log.error("Report was not created for package: {}", ExceptionUtils.getStackTrace(e));
+            }
+        }
+    }
+
+    /**
+     * Формирует результирующий xml файл для конечного пользователя на основе файла из официального сервиса сигналов
+     *
+     * @return Путь к временному рещультирующему файлу
+     */
+    private Path generateResponse(UserPackage userPackage, String reportName, Path loadedReportPath) throws IOException {
+        // Выставляется в true, если хотя бы один субъект с мониторинга подошел под условие срабатывания сигнала. В этом случае отчет сохраняется
+        AtomicBoolean generateResponse = new AtomicBoolean(false);
+
+        String commonMembercode = userPackage.getUserId().substring(0, 6);
+        Path rootReportPath = Path.of(config.getLoadDir(), commonMembercode);
+        if (!Files.exists(rootReportPath)) {
+            Files.createDirectories(rootReportPath);
+        }
+        LocalDateTime newReportDate = CTUtil.parseDateFromReportFileName(reportName);
+        newReportDate = newReportDate == null ? LocalDateTime.now() : newReportDate;
+        AtomicReference<Path> responsePath = new AtomicReference<>(
+            rootReportPath.resolve(
+                String.format("%s_%s_%s.xml",
+                    commonMembercode,
+                    userPackage.getPackageName(),
+                    newReportDate.format(CTUtil.responseReportDateTimeFormatter))
+            ));
+
+        MonitoringSignalResponse response = new MonitoringSignalResponse();
+        //Имя пакета, переданного при установке
+        response.setPackageName(userPackage.getPackageName());
+
+        // Запись в результрующий файл
+        MonitoringSignalResponseStaxXmlWriter staxXmlWriter = new MonitoringSignalResponseStaxXmlWriter(config.getEncoding());
+        staxXmlWriter.writeWithStax(responsePath.toString(), response,
+            writer -> {
+                try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(loadedReportPath), 8 * 1024 * 1024)) {
+                    staxXmlProcessor.read(in, Persons.class,
+                        personsData -> { // как только считали пачку Persons сразу записали ее в результирующий файл
+                            List<Subject> subjectsBatch = processPersons(personsData.getPersons(), userPackage);
+                            if (!subjectsBatch.isEmpty()) {
+                                generateResponse.compareAndSet(false, true);
+                            }
+                            writer.accept(subjectsBatch);
+                            personsData.getPersons().clear(); // чистим список для экономии памяти
+                        },
+                        persons -> persons.getPersons().isEmpty(),
+                        10000,
+                        "/Report/Persons/Person");
+                } finally {
+                    if (!generateResponse.get()) { // ни один субъект не подошел под условие, удаляем временный отчет
+                        CtFileUtils.deleteQuietly(responsePath.get());
+                        responsePath.set(null);
+                    }
+                }
+            });
+
+        CtFileUtils.deleteQuietly(loadedReportPath);
+        return responsePath.get();
+    }
+
+    private List<Subject> processPersons(List<Person> persons, UserPackage userPackage) {
+        List<Subject> subjects = new ArrayList<>();
+        Map<Long, SubjectRepInfo> subjectRepInfoMap = new HashMap<>();
+        for (Person person : persons) {
+            Signals signals = new Signals();
+            for (Inquiry inquiry : person.getInquiries()) {
+                // Отбираем только тех фл, у которых bus_category in (MFO, MKK, MFK) и есть сигналы с кодом 16
+                Integer signalType = CTUtil.checkBusCategory(inquiry.getBusCategory());
+                if (signalType == -1 || inquiry.getSignals().stream().noneMatch(signal -> signal.getN().equals(16)) ||
+                    !userPackage.getSignalTypes().contains(signalType.toString())) {
+                    continue;
+                }
+                Signal signal = new Signal();
+                signal.setSignalType(signalType); // Код типа сигнала по внутреннему справочнику. В текущей версии: 1 = сигнал 16 от MFO/MKK/MFK
+                signal.setInquiryDate(inquiry.getDate()); // Дата и время запроса КИ
+                signal.setInquiryPurpose(inquiry.getInqP()); // Цель запроса КИ
+                signals.getSignals().add(signal);
+            }
+
+            // Если есть хотя бы один заполненный сигнал, значит заполняем остальные данные
+            if (!signals.getSignals().isEmpty()) {
+                PackagesRecordsMap packagesRecord;
+                PhonesMap cacheByPhone;
+                if (subjectRepInfoMap.containsKey(person.getUid())) {
+                    SubjectRepInfo subjectRepInfo = subjectRepInfoMap.get(person.getUid());
+                    packagesRecord = subjectRepInfo.packagesRecord;
+                    cacheByPhone = subjectRepInfo.cacheByPhone;
+                } else {
+                    packagesRecord = packageRecordsService.findPackageRecordsById(person.getUid());
+                    cacheByPhone = packageRecordsService.findCachedByPhone(packagesRecord.getPhone_l());
+                    subjectRepInfoMap.put(person.getUid(), new SubjectRepInfo(packagesRecord, cacheByPhone));
+                }
+                if (cacheByPhone != null) {
+                    Subject subject = new Subject();
+                    subject.setId(packagesRecord.getSubjectId()); // Уникальный идентификатор субъекта (равен Id из XML-пакета при установке мониторинга)
+                    subject.setClientId(CTUtil.normalizePhoneNumberZeroesOnly(packagesRecord.getPhone_l()));
+                    subject.setFlagDebt5000(cacheByPhone.getFlagDebt5000());
+                    subject.setFlag9012(cacheByPhone.getFlag90days());
+                    subject.setSignals(signals);
+                    subjects.add(subject);
+                }
+            }
+        }
+        return subjects;
+    }
+
+    private void updateSignalHits(UserPackage userPackage, Path responsePath, Path reportFileName) {
+        log.info("Updating signal hits for {}", userPackage.getPackageName());
+        LocalDateTime responseSentAt = LocalDateTime.now();
+        // Считываем результирующий файл пачками и обновляем таблицу hits
+        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(responsePath), 8 * 1024 * 1024)) {
+            staxXmlProcessor.read(in, MonitoringSignalResponse.class,
+                response -> { // как только считали пачку Subjects сразу обновляем таблицу hits
+                    List<Hit> hits = new ArrayList<>();
+                    response.getSubjects().getSubjects().forEach(subject -> {
+                        subject.getSignals().getSignals().forEach(signal -> {
+                            if (userPackage.getSignalTypes().contains(signal.getSignalType().toString())) {
+                                Hit hit = new Hit();
+                                hit.setPackageId(userPackage.getId());
+                                hit.setSubjectId(subject.getId());
+                                hit.setPhone(Long.valueOf(subject.getClientId()));
+                                hit.setSignalType(signal.getSignalType());
+                                hit.setResponseSentAt(responseSentAt);
+                                hit.setReportFileName(reportFileName.getFileName().toString());
+                                hits.add(hit);
+                            }
+                        });
+                        if (!hits.isEmpty()) {
+                            hitsService.insertHits(hits);
+                            hits.clear();
+                        }
+                    });
+                    response.getSubjects().getSubjects().clear(); // чистим список для экономии памяти
+                },
+                response -> response.getSubjects().getSubjects().isEmpty()
+                ,
+                10000,
+                "/MonitoringSignalResponse/Subjects/Subject");
+        } catch (IOException e) {
+            throw new RuntimeException("Error while updating hits", e);
+        }
+        CtFileUtils.deleteQuietly(responsePath); // удалям результирующий xml файл с диска (Он уже заархивирован к этому моменту)
+        log.info("Signal hits updated for package {}", userPackage.getPackageName());
+    }
+
+    private class SubjectRepInfo {
+        private final PackagesRecordsMap packagesRecord;
+        private final PhonesMap cacheByPhone;
+
+        private SubjectRepInfo(PackagesRecordsMap packagesRecord, PhonesMap cacheByPhone) {
+            this.packagesRecord = packagesRecord;
+            this.cacheByPhone = cacheByPhone;
+        }
+    }
+}