Snapshot maker created, new task code added to Task.java

clearing-parent/pom.xml

@@ -43,6 +43,7 @@
         <module>swt-importer</module>
         <module>gateway-api</module>
         <module>imdg-hist</module>
+        <module>snapshot-maker</module>
     </modules>
 
     <properties>

clearing-parent/snapshot-maker/pom.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>snapshot-maker</artifactId>
+    <name>snapshot-maker</name>
+    <description>Database snapshot maker</description>
+    <version>SPCEX-3.12.5</version>
+
+    <parent>
+        <groupId>ru.spcex.clearing</groupId>
+        <artifactId>clearing-parent</artifactId>
+        <version>SPCEX-3.12.5</version>
+    </parent>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+
+        <hikaricp.version>5.1.0</hikaricp.version>
+        <springdoc-openapi.version>2.5.0</springdoc-openapi.version>
+        <commons-io.version>2.16.1</commons-io.version>
+    </properties>
+
+    <dependencies>
+        <!-- Spring dependencies -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+        <!-- Kafka dependencies -->
+        <dependency>
+            <groupId>ru.spcex.platform</groupId>
+            <artifactId>platform-messaging</artifactId>
+        </dependency>
+
+        <!-- Misc dependencies -->
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>${commons-io.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.clearing</groupId>
+            <artifactId>classes</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.platform</groupId>
+            <artifactId>platform-enum</artifactId>
+        </dependency>
+
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <excludes>
+                    <exclude>application.properties</exclude>
+                </excludes>
+                <filtering>false</filtering>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <finalName>${project.artifactId}</finalName>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/SnapshotMakerApplication.java

@@ -0,0 +1,11 @@
+package ru.spcex.clearing.snapshot.maker;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class SnapshotMakerApplication {
+    public static void main(String[] args) {
+        SpringApplication.run(SnapshotMakerApplication.class, args);
+    }
+}

clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/KafkaConfig.java

@@ -0,0 +1,32 @@
+package ru.spcex.clearing.snapshot.maker.config;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.producer.Producer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Scope;
+import ru.spcex.clearing.platform.messaging.config.KafkaConsumerFactory;
+import ru.spcex.clearing.platform.messaging.config.KafkaProducerFactory;
+import ru.spcex.clearing.snapshot.maker.config.settings.SnapshotMakerSettings;
+
+@Configuration
+public class KafkaConfig {
+    @Autowired
+    @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+    @Bean
+    public Consumer<String, Object> createConsumer(SnapshotMakerSettings settings) {
+        return KafkaConsumerFactory.consumer(settings.getKafkaConsumer());
+    }
+
+    @Autowired
+    @Bean
+    public Producer<String, Object> createProducer(SnapshotMakerSettings settings) {
+        if (settings.getKafkaProducer() != null) {
+            return KafkaProducerFactory.producer(settings.getKafkaProducer());
+        } else {
+            return null;
+        }
+    }
+}

clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/settings/DatabaseSettings.java

@@ -0,0 +1,31 @@
+package ru.spcex.clearing.snapshot.maker.config.settings;
+
+public class DatabaseSettings {
+    private String jdbcUrl;
+    private String username;
+    private String password;
+
+    public String getJdbcUrl() {
+        return jdbcUrl;
+    }
+
+    public void setJdbcUrl(String jdbcUrl) {
+        this.jdbcUrl = jdbcUrl;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+}

clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/settings/LocationSettings.java

@@ -0,0 +1,67 @@
+package ru.spcex.clearing.snapshot.maker.config.settings;
+
+public class LocationSettings {
+    private String localStore;
+    private String dbfImporterFolder;
+    private String dbfExporterFolder;
+    private String swtImporterFolder;
+    private String swtExporterFolder;
+    private String xmlImporterFolder;
+    private String xmlExporterFolder;
+
+    public String getLocalStore() {
+        return localStore;
+    }
+
+    public void setLocalStore(String localStore) {
+        this.localStore = localStore;
+    }
+
+    public String getDbfImporterFolder() {
+        return dbfImporterFolder;
+    }
+
+    public void setDbfImporterFolder(String dbfImporterFolder) {
+        this.dbfImporterFolder = dbfImporterFolder;
+    }
+
+    public String getDbfExporterFolder() {
+        return dbfExporterFolder;
+    }
+
+    public void setDbfExporterFolder(String dbfExporterFolder) {
+        this.dbfExporterFolder = dbfExporterFolder;
+    }
+
+    public String getSwtImporterFolder() {
+        return swtImporterFolder;
+    }
+
+    public void setSwtImporterFolder(String swtImporterFolder) {
+        this.swtImporterFolder = swtImporterFolder;
+    }
+
+    public String getSwtExporterFolder() {
+        return swtExporterFolder;
+    }
+
+    public void setSwtExporterFolder(String swtExporterFolder) {
+        this.swtExporterFolder = swtExporterFolder;
+    }
+
+    public String getXmlImporterFolder() {
+        return xmlImporterFolder;
+    }
+
+    public void setXmlImporterFolder(String xmlImporterFolder) {
+        this.xmlImporterFolder = xmlImporterFolder;
+    }
+
+    public String getXmlExporterFolder() {
+        return xmlExporterFolder;
+    }
+
+    public void setXmlExporterFolder(String xmlExporterFolder) {
+        this.xmlExporterFolder = xmlExporterFolder;
+    }
+}

clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/settings/SnapshotMakerSettings.java

@@ -0,0 +1,49 @@
+package ru.spcex.clearing.snapshot.maker.config.settings;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
+import ru.spcex.clearing.platform.messaging.config.element.KafkaConsumerSettings;
+import ru.spcex.clearing.platform.messaging.config.element.KafkaProducerSettings;
+
+@Component
+@PropertySource("file:${spring.config.location}/application.properties")
+@ConfigurationProperties("snapshot-maker")
+public class SnapshotMakerSettings {
+    private DatabaseSettings database;
+    private KafkaConsumerSettings kafkaConsumer;
+    private KafkaProducerSettings kafkaProducer;
+    private LocationSettings location;
+
+    public DatabaseSettings getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(DatabaseSettings database) {
+        this.database = database;
+    }
+
+    public KafkaConsumerSettings getKafkaConsumer() {
+        return kafkaConsumer;
+    }
+
+    public void setKafkaConsumer(KafkaConsumerSettings kafkaConsumer) {
+        this.kafkaConsumer = kafkaConsumer;
+    }
+
+    public KafkaProducerSettings getKafkaProducer() {
+        return kafkaProducer;
+    }
+
+    public void setKafkaProducer(KafkaProducerSettings kafkaProducer) {
+        this.kafkaProducer = kafkaProducer;
+    }
+
+    public LocationSettings getLocation() {
+        return location;
+    }
+
+    public void setLocation(LocationSettings location) {
+        this.location = location;
+    }
+}

clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/service/CommandService.java

@@ -0,0 +1,52 @@
+package ru.spcex.clearing.snapshot.maker.service;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.producer.Producer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Service;
+import ru.spcex.clearing.platform.messaging.domain.BaseRequest;
+import ru.spcex.clearing.platform.messaging.domain.Consts;
+import ru.spcex.clearing.platform.messaging.domain.cud.schedule.LauncherCommandRequest;
+import ru.spcex.clearing.platform.messaging.service.QueueConsumer;
+import ru.spcex.clearing.platform.messaging.service.RequestInfoUpdate;
+import ru.spcex.clearing.platform.messaging.service.Status;
+
+@Service
+public class CommandService extends QueueConsumer implements InitializingBean {
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    private final SessionService sessionService;
+
+    public CommandService(Consumer<String, Object> kafkaQueue,
+                          Producer<String, Object> kafkaResponseQueue,
+                          SessionService sessionService) {
+        super(kafkaQueue, kafkaResponseQueue);
+        this.sessionService = sessionService;
+    }
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        callback(LauncherCommandRequest.class)
+            .setFunction(this::process)
+            .forDestination(Consts.LAUNCHER_NEW, callbacks::put);
+
+        init();
+    }
+
+    private RequestInfoUpdate process(BaseRequest<LauncherCommandRequest> request) {
+        RequestInfoUpdate requestInfoUpdate = new RequestInfoUpdate();
+
+        if (request.getRequestPayload().getTaskName().equals("STSS")) {
+            requestInfoUpdate = sessionService.startSession();
+        } else if (request.getRequestPayload().getTaskName().equals("SPSS")) {
+            requestInfoUpdate = sessionService.stopSession();
+        } else {
+            log.info("Unknown task: {}", request.getRequestPayload().getTaskName());
+            requestInfoUpdate.setStatus(Status.Error);
+            requestInfoUpdate.setMessage("Unknown task: " + request.getRequestPayload().getTaskName());
+        }
+
+        return requestInfoUpdate;
+    }
+}

clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/service/SessionService.java

@@ -0,0 +1,167 @@
+package ru.spcex.clearing.snapshot.maker.service;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import ru.spcex.clearing.platform.messaging.service.RequestInfoUpdate;
+import ru.spcex.clearing.platform.messaging.service.Status;
+import ru.spcex.clearing.snapshot.maker.config.settings.SnapshotMakerSettings;
+
+@Service
+public class SessionService {
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    private final SnapshotMakerSettings settings;
+    private final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH-mm-ss");
+    private final static Pattern JDBC_URL_PATTERN = Pattern.compile("(?<=//)(.+?)(?=:):(.+?)(?=/)/(.+?)(?=\\?|$)");
+    private Path currentPath;
+
+    public SessionService(SnapshotMakerSettings settings) {
+        this.settings = settings;
+    }
+
+    public RequestInfoUpdate startSession() {
+        RequestInfoUpdate requestInfoUpdate = new RequestInfoUpdate();
+
+        if (currentPath != null) {
+            log.info("You need to stop session first!");
+            requestInfoUpdate.setStatus(Status.Error);
+            requestInfoUpdate.setMessage("You need to stop session first!");
+            return requestInfoUpdate;
+        }
+
+        log.info("Starting making snapshot");
+
+        try {
+            this.currentPath = Files.createDirectories(Paths.get(settings.getLocation().getLocalStore()).resolve(LocalDateTime.now().format(dtf)));
+            log.info("Session folder created: {}", this.currentPath);
+        } catch (IOException e) {
+            log.info("Can't create directory, error={}", e.getMessage());
+            requestInfoUpdate.setStatus(Status.Error);
+            requestInfoUpdate.setMessage("Can't create directory, error = " + e.getMessage());
+            return requestInfoUpdate;
+        }
+        dumpDatabase(requestInfoUpdate);
+
+        if (requestInfoUpdate.getStatus() == Status.Error) {
+            return requestInfoUpdate;
+        }
+
+        requestInfoUpdate.setStatus(Status.Success);
+        requestInfoUpdate.setMessage("Snapshot created successfully in folder " + this.currentPath.toAbsolutePath());
+        return requestInfoUpdate;
+    }
+
+    public RequestInfoUpdate stopSession() {
+        RequestInfoUpdate requestInfoUpdate = new RequestInfoUpdate();
+
+        if (currentPath == null) {
+            log.info("You need to start session first!");
+            requestInfoUpdate.setStatus(Status.Error);
+            requestInfoUpdate.setMessage("You need to start session first!");
+            return requestInfoUpdate;
+        }
+
+        log.info("Stopping session");
+
+        copyDirectories(requestInfoUpdate);
+        this.currentPath = null;
+
+        if (requestInfoUpdate.getStatus() == Status.Error) {
+            return requestInfoUpdate;
+        }
+
+        requestInfoUpdate.setStatus(Status.Success);
+        requestInfoUpdate.setMessage("Session snapshot stopped successfully!");
+        return requestInfoUpdate;
+    }
+
+    private void dumpDatabase(RequestInfoUpdate requestInfoUpdate) {
+        Matcher matcher = JDBC_URL_PATTERN.matcher(settings.getDatabase().getJdbcUrl());
+        if (!matcher.find()) {
+            requestInfoUpdate.setStatus(Status.Error);
+            requestInfoUpdate.setMessage("Database URL is not valid!");
+            return;
+        }
+
+        Path dumpLocation = this.currentPath.resolve("database_dump.sql");
+
+        ProcessBuilder pb = new ProcessBuilder();
+        pb.environment().put("PGPASSWORD", settings.getDatabase().getPassword());
+
+        if (System.getProperty("os.name").contains("Windows")) {
+            pb.command("cmd", "/c", "pg_dump -U " + settings.getDatabase().getUsername() + " -h " + matcher.group(1) + " -p " + matcher.group(2) + " -d " + matcher.group(3) + " --column-inserts -f " + dumpLocation.toAbsolutePath());
+        } else {
+            pb.command("/bin/sh", "-c", "pg_dump -U " + settings.getDatabase().getUsername() + " -h " + matcher.group(1) + " -p " + matcher.group(2) + " -d " + matcher.group(3) + " --column-inserts -f " + dumpLocation.toAbsolutePath());
+        }
+
+        try {
+            log.info("Executing command {}", pb.command());
+            Process process = pb.start();
+            StringBuilder out = new StringBuilder();
+            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+
+            String line;
+            while ((line = reader.readLine()) != null) {
+                out.append(line).append(System.lineSeparator());
+            }
+            int statusCode = process.waitFor();
+            log.info("Status code={}, output {}", statusCode, out);
+        } catch (IOException | InterruptedException e) {
+            log.info("Can't dump database, error={}", e.getMessage());
+            requestInfoUpdate.setStatus(Status.Error);
+            requestInfoUpdate.setMessage("Can't dump database, error = " + e.getMessage());
+        }
+    }
+
+    private void copyDirectories(RequestInfoUpdate requestInfoUpdate) {
+        List<Path> directoriesToCopy = List.of(
+            Paths.get(settings.getLocation().getDbfImporterFolder()),
+            Paths.get(settings.getLocation().getDbfExporterFolder()),
+            Paths.get(settings.getLocation().getSwtImporterFolder()),
+            Paths.get(settings.getLocation().getSwtExporterFolder()),
+            Paths.get(settings.getLocation().getXmlImporterFolder()),
+            Paths.get(settings.getLocation().getXmlExporterFolder())
+        );
+
+        for (Path directory : directoriesToCopy) {
+            if (!Files.exists(directory) || !Files.isDirectory(directory)) {
+                log.info("{} is not a directory or does not exist.", directory.toAbsolutePath());
+                requestInfoUpdate.setStatus(Status.Error);
+                requestInfoUpdate.setMessage(directory.toAbsolutePath() + " is not a directory or does not exist.");
+                return;
+            }
+
+            Path target;
+            try {
+                target = Files.createDirectories(this.currentPath.resolve(directory.getParent().getFileName().toString()).resolve(directory.getFileName().toString()));
+            } catch (IOException e) {
+                log.info("Can't create directory, error={}", e.getMessage());
+                requestInfoUpdate.setStatus(Status.Error);
+                requestInfoUpdate.setMessage("Can't create directory, error = " + e.getMessage());
+                return;
+            }
+
+            try {
+                log.info("Copying {} to {}", directory.toAbsolutePath(), target.toAbsolutePath());
+                FileUtils.copyDirectory(directory.toFile(), target.toFile());
+            } catch (IOException e) {
+                log.info("Can't copy directory {}, error={}", directory, e.getMessage());
+                requestInfoUpdate.setStatus(Status.Error);
+                requestInfoUpdate.setMessage("Can't copy directory " + directory + ", error = " + e.getMessage());
+                return;
+            }
+        }
+    }
+}

clearing-parent/snapshot-maker/src/main/resources/application.properties

@@ -0,0 +1,30 @@
+# Database settings
+snapshot-maker.database.jdbcUrl=jdbc:postgresql://localhost:5433/clearing
+snapshot-maker.database.username=clearing
+snapshot-maker.database.password=Aa111111
+
+# Locations setting
+snapshot-maker.location.local-store=/mnt/c/Users/ivan/Desktop/snapshot-test/out
+snapshot-maker.location.dbf-importer-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/dbf/importer
+snapshot-maker.location.dbf-exporter-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/dbf/exporter
+snapshot-maker.location.swt-importer-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/swt/importer
+snapshot-maker.location.swt-exporter-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/swt/exporter
+snapshot-maker.location.xml-importer-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/xml/importer
+snapshot-maker.location.xml-exporter-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/xml/exporter
+
+# Kafka producer settings
+snapshot-maker.kafka-producer.bootstrap-servers=localhost:9092
+snapshot-maker.kafka-producer.acks=all
+snapshot-maker.kafka-producer.retries=0
+snapshot-maker.kafka-producer.batch-size=16384
+snapshot-maker.kafka-producer.linger-ms=1
+snapshot-maker.kafka-producer.buffer-memory=33554432
+
+# Kafka consumer settings
+snapshot-maker.kafka-consumer.bootstrap-servers=localhost:9092
+snapshot-maker.kafka-consumer.group-id=dev-group-clearing-service
+snapshot-maker.kafka-consumer.enable-auto-commit=false
+snapshot-maker.kafka-consumer.session-timeout-ms=30000
+snapshot-maker.kafka-consumer.auto-offset-reset=latest
+snapshot-maker.kafka-consumer.linger-ms=1
+snapshot-maker.kafka-consumer.buffer-memory=33554432

platform-parent/platform-enum/src/main/java/ru/spcex/platform/enumeration/Task.java

@@ -59,6 +59,8 @@ public enum Task implements IEnumKey {
     finishBadSessions("CCLR"),//Завершение неудачных клиринговых сессий
     sendLim_LIMC("LIMC"), // Выгрузка в Торговую систему остатков по валюте (отправка lim)"
     makeFiles_MTCR("MTCR"), // Формирование файлов с МТКР
+    startSessionSnapshot("STSS"), // Старт формирования снэпшота сессии
+    stopSessionSnapshot("SPSS"), // Завершение формирования снэпшота сессии
     ;
 
     private final String key;

pom.xml

@@ -53,6 +53,7 @@
         <folder_root_registry-service>${folder_root_clearing}/clearing-parent/registry-service</folder_root_registry-service>
         <folder_root_scheduler-service>${folder_root_clearing}/clearing-parent/scheduler-service</folder_root_scheduler-service>
         <folder_root_gateway-api>${folder_root_clearing}/clearing-parent/gateway-api</folder_root_gateway-api>
+        <folder_root_snapshot-maker>${folder_root_clearing}/clearing-parent/snapshot-maker</folder_root_snapshot-maker>
         <!-- IMDG -->
         <external_libraries.hazelcast.version>3.12.4</external_libraries.hazelcast.version>
         <external_libraries.slf4j.version>1.7.33</external_libraries.slf4j.version>