treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Snapshot maker created, new task code added to Task.java

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2024-07-04 16:21:43 +03:00
parent 1df96b7ef6
commit 95f7741111
12 changed files with 534 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
clearing-parent/pom.xml
View file

@ -43,6 +43,7 @@
<module>swt-importer</module>
<module>gateway-api</module>
<module>imdg-hist</module>
<module>snapshot-maker</module>
</modules>
<properties>

91
clearing-parent/snapshot-maker/pom.xml Normal file
View file

@ -0,0 +1,91 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>snapshot-maker</artifactId>
<name>snapshot-maker</name>
<description>Database snapshot maker</description>
<version>SPCEX-3.12.5</version>
<parent>
<groupId>ru.spcex.clearing</groupId>
<artifactId>clearing-parent</artifactId>
<version>SPCEX-3.12.5</version>
</parent>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<hikaricp.version>5.1.0</hikaricp.version>
<springdoc-openapi.version>2.5.0</springdoc-openapi.version>
<commons-io.version>2.16.1</commons-io.version>
</properties>
<dependencies>
<!-- Spring dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- Kafka dependencies -->
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-messaging</artifactId>
</dependency>
<!-- Misc dependencies -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>classes</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-enum</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>application.properties</exclude>
</excludes>
<filtering>false</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
<configuration>
<finalName>${project.artifactId}</finalName>
</configuration>
</plugin>
</plugins>
</build>
</project>

11
clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/SnapshotMakerApplication.java Normal file
View file

@ -0,0 +1,11 @@
package ru.spcex.clearing.snapshot.maker;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class SnapshotMakerApplication {
public static void main(String[] args) {
SpringApplication.run(SnapshotMakerApplication.class, args);
}
}

32
clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/KafkaConfig.java Normal file
View file

@ -0,0 +1,32 @@
package ru.spcex.clearing.snapshot.maker.config;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.producer.Producer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
import ru.spcex.clearing.platform.messaging.config.KafkaConsumerFactory;
import ru.spcex.clearing.platform.messaging.config.KafkaProducerFactory;
import ru.spcex.clearing.snapshot.maker.config.settings.SnapshotMakerSettings;
@Configuration
public class KafkaConfig {
@Autowired
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
@Bean
public Consumer<String, Object> createConsumer(SnapshotMakerSettings settings) {
return KafkaConsumerFactory.consumer(settings.getKafkaConsumer());
}
@Autowired
@Bean
public Producer<String, Object> createProducer(SnapshotMakerSettings settings) {
if (settings.getKafkaProducer() != null) {
return KafkaProducerFactory.producer(settings.getKafkaProducer());
} else {
return null;
}
}
}

31
clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/settings/DatabaseSettings.java Normal file
View file

@ -0,0 +1,31 @@
package ru.spcex.clearing.snapshot.maker.config.settings;
public class DatabaseSettings {
private String jdbcUrl;
private String username;
private String password;
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
}

67
clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/settings/LocationSettings.java Normal file
View file

@ -0,0 +1,67 @@
package ru.spcex.clearing.snapshot.maker.config.settings;
public class LocationSettings {
private String localStore;
private String dbfImporterFolder;
private String dbfExporterFolder;
private String swtImporterFolder;
private String swtExporterFolder;
private String xmlImporterFolder;
private String xmlExporterFolder;
public String getLocalStore() {
return localStore;
}
public void setLocalStore(String localStore) {
this.localStore = localStore;
}
public String getDbfImporterFolder() {
return dbfImporterFolder;
}
public void setDbfImporterFolder(String dbfImporterFolder) {
this.dbfImporterFolder = dbfImporterFolder;
}
public String getDbfExporterFolder() {
return dbfExporterFolder;
}
public void setDbfExporterFolder(String dbfExporterFolder) {
this.dbfExporterFolder = dbfExporterFolder;
}
public String getSwtImporterFolder() {
return swtImporterFolder;
}
public void setSwtImporterFolder(String swtImporterFolder) {
this.swtImporterFolder = swtImporterFolder;
}
public String getSwtExporterFolder() {
return swtExporterFolder;
}
public void setSwtExporterFolder(String swtExporterFolder) {
this.swtExporterFolder = swtExporterFolder;
}
public String getXmlImporterFolder() {
return xmlImporterFolder;
}
public void setXmlImporterFolder(String xmlImporterFolder) {
this.xmlImporterFolder = xmlImporterFolder;
}
public String getXmlExporterFolder() {
return xmlExporterFolder;
}
public void setXmlExporterFolder(String xmlExporterFolder) {
this.xmlExporterFolder = xmlExporterFolder;
}
}

49
clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/config/settings/SnapshotMakerSettings.java Normal file
View file

@ -0,0 +1,49 @@
package ru.spcex.clearing.snapshot.maker.config.settings;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.platform.messaging.config.element.KafkaConsumerSettings;
import ru.spcex.clearing.platform.messaging.config.element.KafkaProducerSettings;
@Component
@PropertySource("file:${spring.config.location}/application.properties")
@ConfigurationProperties("snapshot-maker")
public class SnapshotMakerSettings {
private DatabaseSettings database;
private KafkaConsumerSettings kafkaConsumer;
private KafkaProducerSettings kafkaProducer;
private LocationSettings location;
public DatabaseSettings getDatabase() {
return database;
}
public void setDatabase(DatabaseSettings database) {
this.database = database;
}
public KafkaConsumerSettings getKafkaConsumer() {
return kafkaConsumer;
}
public void setKafkaConsumer(KafkaConsumerSettings kafkaConsumer) {
this.kafkaConsumer = kafkaConsumer;
}
public KafkaProducerSettings getKafkaProducer() {
return kafkaProducer;
}
public void setKafkaProducer(KafkaProducerSettings kafkaProducer) {
this.kafkaProducer = kafkaProducer;
}
public LocationSettings getLocation() {
return location;
}
public void setLocation(LocationSettings location) {
this.location = location;
}
}

52
clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/service/CommandService.java Normal file
View file

@ -0,0 +1,52 @@
package ru.spcex.clearing.snapshot.maker.service;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.producer.Producer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Service;
import ru.spcex.clearing.platform.messaging.domain.BaseRequest;
import ru.spcex.clearing.platform.messaging.domain.Consts;
import ru.spcex.clearing.platform.messaging.domain.cud.schedule.LauncherCommandRequest;
import ru.spcex.clearing.platform.messaging.service.QueueConsumer;
import ru.spcex.clearing.platform.messaging.service.RequestInfoUpdate;
import ru.spcex.clearing.platform.messaging.service.Status;
@Service
public class CommandService extends QueueConsumer implements InitializingBean {
private final Logger log = LoggerFactory.getLogger(getClass());
private final SessionService sessionService;
public CommandService(Consumer<String, Object> kafkaQueue,
Producer<String, Object> kafkaResponseQueue,
SessionService sessionService) {
super(kafkaQueue, kafkaResponseQueue);
this.sessionService = sessionService;
}
@Override
public void afterPropertiesSet() throws Exception {
callback(LauncherCommandRequest.class)
.setFunction(this::process)
.forDestination(Consts.LAUNCHER_NEW, callbacks::put);
init();
}
private RequestInfoUpdate process(BaseRequest<LauncherCommandRequest> request) {
RequestInfoUpdate requestInfoUpdate = new RequestInfoUpdate();
if (request.getRequestPayload().getTaskName().equals("STSS")) {
requestInfoUpdate = sessionService.startSession();
} else if (request.getRequestPayload().getTaskName().equals("SPSS")) {
requestInfoUpdate = sessionService.stopSession();
} else {
log.info("Unknown task: {}", request.getRequestPayload().getTaskName());
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("Unknown task: " + request.getRequestPayload().getTaskName());
}
return requestInfoUpdate;
}
}

167
clearing-parent/snapshot-maker/src/main/java/ru/spcex/clearing/snapshot/maker/service/SessionService.java Normal file
View file

@ -0,0 +1,167 @@
package ru.spcex.clearing.snapshot.maker.service;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import ru.spcex.clearing.platform.messaging.service.RequestInfoUpdate;
import ru.spcex.clearing.platform.messaging.service.Status;
import ru.spcex.clearing.snapshot.maker.config.settings.SnapshotMakerSettings;
@Service
public class SessionService {
private final Logger log = LoggerFactory.getLogger(getClass());
private final SnapshotMakerSettings settings;
private final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH-mm-ss");
private final static Pattern JDBC_URL_PATTERN = Pattern.compile("(?<=//)(.+?)(?=:):(.+?)(?=/)/(.+?)(?=\\?|$)");
private Path currentPath;
public SessionService(SnapshotMakerSettings settings) {
this.settings = settings;
}
public RequestInfoUpdate startSession() {
RequestInfoUpdate requestInfoUpdate = new RequestInfoUpdate();
if (currentPath != null) {
log.info("You need to stop session first!");
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("You need to stop session first!");
return requestInfoUpdate;
}
log.info("Starting making snapshot");
try {
this.currentPath = Files.createDirectories(Paths.get(settings.getLocation().getLocalStore()).resolve(LocalDateTime.now().format(dtf)));
log.info("Session folder created: {}", this.currentPath);
} catch (IOException e) {
log.info("Can't create directory, error={}", e.getMessage());
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("Can't create directory, error = " + e.getMessage());
return requestInfoUpdate;
}
dumpDatabase(requestInfoUpdate);
if (requestInfoUpdate.getStatus() == Status.Error) {
return requestInfoUpdate;
}
requestInfoUpdate.setStatus(Status.Success);
requestInfoUpdate.setMessage("Snapshot created successfully in folder " + this.currentPath.toAbsolutePath());
return requestInfoUpdate;
}
public RequestInfoUpdate stopSession() {
RequestInfoUpdate requestInfoUpdate = new RequestInfoUpdate();
if (currentPath == null) {
log.info("You need to start session first!");
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("You need to start session first!");
return requestInfoUpdate;
}
log.info("Stopping session");
copyDirectories(requestInfoUpdate);
this.currentPath = null;
if (requestInfoUpdate.getStatus() == Status.Error) {
return requestInfoUpdate;
}
requestInfoUpdate.setStatus(Status.Success);
requestInfoUpdate.setMessage("Session snapshot stopped successfully!");
return requestInfoUpdate;
}
private void dumpDatabase(RequestInfoUpdate requestInfoUpdate) {
Matcher matcher = JDBC_URL_PATTERN.matcher(settings.getDatabase().getJdbcUrl());
if (!matcher.find()) {
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("Database URL is not valid!");
return;
}
Path dumpLocation = this.currentPath.resolve("database_dump.sql");
ProcessBuilder pb = new ProcessBuilder();
pb.environment().put("PGPASSWORD", settings.getDatabase().getPassword());
if (System.getProperty("os.name").contains("Windows")) {
pb.command("cmd", "/c", "pg_dump -U " + settings.getDatabase().getUsername() + " -h " + matcher.group(1) + " -p " + matcher.group(2) + " -d " + matcher.group(3) + " --column-inserts -f " + dumpLocation.toAbsolutePath());
} else {
pb.command("/bin/sh", "-c", "pg_dump -U " + settings.getDatabase().getUsername() + " -h " + matcher.group(1) + " -p " + matcher.group(2) + " -d " + matcher.group(3) + " --column-inserts -f " + dumpLocation.toAbsolutePath());
}
try {
log.info("Executing command {}", pb.command());
Process process = pb.start();
StringBuilder out = new StringBuilder();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
out.append(line).append(System.lineSeparator());
}
int statusCode = process.waitFor();
log.info("Status code={}, output {}", statusCode, out);
} catch (IOException | InterruptedException e) {
log.info("Can't dump database, error={}", e.getMessage());
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("Can't dump database, error = " + e.getMessage());
}
}
private void copyDirectories(RequestInfoUpdate requestInfoUpdate) {
List<Path> directoriesToCopy = List.of(
Paths.get(settings.getLocation().getDbfImporterFolder()),
Paths.get(settings.getLocation().getDbfExporterFolder()),
Paths.get(settings.getLocation().getSwtImporterFolder()),
Paths.get(settings.getLocation().getSwtExporterFolder()),
Paths.get(settings.getLocation().getXmlImporterFolder()),
Paths.get(settings.getLocation().getXmlExporterFolder())
);
for (Path directory : directoriesToCopy) {
if (!Files.exists(directory) || !Files.isDirectory(directory)) {
log.info("{} is not a directory or does not exist.", directory.toAbsolutePath());
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage(directory.toAbsolutePath() + " is not a directory or does not exist.");
return;
}
Path target;
try {
target = Files.createDirectories(this.currentPath.resolve(directory.getParent().getFileName().toString()).resolve(directory.getFileName().toString()));
} catch (IOException e) {
log.info("Can't create directory, error={}", e.getMessage());
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("Can't create directory, error = " + e.getMessage());
return;
}
try {
log.info("Copying {} to {}", directory.toAbsolutePath(), target.toAbsolutePath());
FileUtils.copyDirectory(directory.toFile(), target.toFile());
} catch (IOException e) {
log.info("Can't copy directory {}, error={}", directory, e.getMessage());
requestInfoUpdate.setStatus(Status.Error);
requestInfoUpdate.setMessage("Can't copy directory " + directory + ", error = " + e.getMessage());
return;
}
}
}
}

30
clearing-parent/snapshot-maker/src/main/resources/application.properties Normal file
View file

@ -0,0 +1,30 @@
# Database settings
snapshot-maker.database.jdbcUrl=jdbc:postgresql://localhost:5433/clearing
snapshot-maker.database.username=clearing
snapshot-maker.database.password=Aa111111
# Locations setting
snapshot-maker.location.local-store=/mnt/c/Users/ivan/Desktop/snapshot-test/out
snapshot-maker.location.dbf-importer-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/dbf/importer
snapshot-maker.location.dbf-exporter-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/dbf/exporter
snapshot-maker.location.swt-importer-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/swt/importer
snapshot-maker.location.swt-exporter-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/swt/exporter
snapshot-maker.location.xml-importer-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/xml/importer
snapshot-maker.location.xml-exporter-folder=/mnt/c/Users/ivan/Desktop/snapshot-test/in/xml/exporter
# Kafka producer settings
snapshot-maker.kafka-producer.bootstrap-servers=localhost:9092
snapshot-maker.kafka-producer.acks=all
snapshot-maker.kafka-producer.retries=0
snapshot-maker.kafka-producer.batch-size=16384
snapshot-maker.kafka-producer.linger-ms=1
snapshot-maker.kafka-producer.buffer-memory=33554432
# Kafka consumer settings
snapshot-maker.kafka-consumer.bootstrap-servers=localhost:9092
snapshot-maker.kafka-consumer.group-id=dev-group-clearing-service
snapshot-maker.kafka-consumer.enable-auto-commit=false
snapshot-maker.kafka-consumer.session-timeout-ms=30000
snapshot-maker.kafka-consumer.auto-offset-reset=latest
snapshot-maker.kafka-consumer.linger-ms=1
snapshot-maker.kafka-consumer.buffer-memory=33554432

2
platform-parent/platform-enum/src/main/java/ru/spcex/platform/enumeration/Task.java
View file

@ -59,6 +59,8 @@ public enum Task implements IEnumKey {
finishBadSessions("CCLR"),//Завершение неудачных клиринговых сессий
sendLim_LIMC("LIMC"), // Выгрузка в Торговую систему остатков по валюте (отправка lim)"
makeFiles_MTCR("MTCR"), // Формирование файлов с МТКР
startSessionSnapshot("STSS"), // Старт формирования снэпшота сессии
stopSessionSnapshot("SPSS"), // Завершение формирования снэпшота сессии
;
private final String key;

1
pom.xml
View file

@ -53,6 +53,7 @@
<folder_root_registry-service>${folder_root_clearing}/clearing-parent/registry-service</folder_root_registry-service>
<folder_root_scheduler-service>${folder_root_clearing}/clearing-parent/scheduler-service</folder_root_scheduler-service>
<folder_root_gateway-api>${folder_root_clearing}/clearing-parent/gateway-api</folder_root_gateway-api>
<folder_root_snapshot-maker>${folder_root_clearing}/clearing-parent/snapshot-maker</folder_root_snapshot-maker>
<!-- IMDG -->
<external_libraries.hazelcast.version>3.12.4</external_libraries.hazelcast.version>
<external_libraries.slf4j.version>1.7.33</external_libraries.slf4j.version>