add new module csv-importer

clearing-parent/csv-importer/pom.xml

@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>ru.spcex.clearing</groupId>
+        <artifactId>clearing-parent</artifactId>
+        <version>SPCEX-3.23.295</version>
+    </parent>
+
+    <artifactId>csv-importer</artifactId>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>ru.spcex.platform</groupId>
+            <artifactId>platform-messaging</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.platform</groupId>
+            <artifactId>platform-imdg-api-hazelcast-impl</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.clearing</groupId>
+            <artifactId>security-util</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.clearing</groupId>
+            <artifactId>classes</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.platform</groupId>
+            <artifactId>platform-enum</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.clearing</groupId>
+            <artifactId>clearing-validation</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <!-- special logging -->
+        <dependency>
+            <groupId>net.logstash.logback</groupId>
+            <artifactId>logstash-logback-encoder</artifactId>
+            <version>7.0.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>ru.spcex.clearing</groupId>
+            <artifactId>test-clearing</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <excludes>
+                    <exclude>application.properties</exclude>
+                </excludes>
+                <filtering>false</filtering>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <finalName>${project.artifactId}</finalName>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.21.0</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.junit.platform</groupId>
+                        <artifactId>junit-platform-surefire-provider</artifactId>
+                        <version>1.2.0-M1</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.junit.jupiter</groupId>
+                        <artifactId>junit-jupiter-engine</artifactId>
+                        <version>5.2.0-M1</version>
+                    </dependency>
+                </dependencies>
+            </plugin>
+        </plugins>
+
+    </build>
+</project>

clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/CsvImporterApplication.java

@@ -0,0 +1,18 @@
+package ru.spcex.clearing.csv.importer;
+
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+
+@SpringBootApplication
+public class CsvImporterApplication {
+    public static void main(String[] args) {
+        try {
+            SpringApplicationBuilder builder = new SpringApplicationBuilder(CsvImporterApplication.class);
+            builder.run(args);
+        } catch (Exception e) {
+            LoggerFactory.getLogger(CsvImporterApplication.class).error("csv-importer start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage());
+            System.exit(-1);
+        }
+    }
+}

clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/AppConfig.java

@@ -0,0 +1,47 @@
+package ru.spcex.clearing.csv.importer.config;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import ru.spcex.clearing.csv.importer.config.settings.CsvImporterSettings;
+import ru.spcex.platform.imdg.api.ImdgProvider;
+import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
+
+@Configuration
+public class AppConfig {
+
+    private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) {
+        ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
+        if (maxPoolSz > 2) {
+            pool.setKeepAliveSeconds(60);
+            pool.setAllowCoreThreadTimeOut(true);
+        }
+        pool.setCorePoolSize(maxPoolSz);
+        pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion);
+        return pool;
+    }
+
+    @Bean(name = "taskExecutorHazelcastClientInitializer")
+    public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() {
+        return createThreadPoolTaskExecutor(1, true);
+    }
+
+    @Bean(name = "taskExecutorIdGeneratorAwaiter")
+    public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() {
+        return createThreadPoolTaskExecutor(1, false);
+    }
+
+    @Autowired
+    @Bean
+    public ImdgProvider imdgProvider(
+        @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
+        @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
+        CsvImporterSettings settings
+    ) {
+        return new HazelcastService(taskExecutorHazelcastClientInitializer,
+            taskExecutorIdGeneratorAwaiter,
+            settings.getHazelcast());
+    }
+}

clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/settings/CsvImporterSettings.java

@@ -0,0 +1,41 @@
+package ru.spcex.clearing.csv.importer.config.settings;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
+import ru.spcex.clearing.platform.messaging.config.element.KafkaConsumerSettings;
+import ru.spcex.clearing.platform.messaging.config.element.KafkaProducerSettings;
+import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams;
+
+@Component
+@PropertySource("file:${spring.config.location}/application.properties")
+@ConfigurationProperties("csv-importer")
+public class CsvImporterSettings {
+    private HazelcastClientParams hazelcast;
+    private KafkaConsumerSettings kafkaConsumer;
+    private KafkaProducerSettings kafkaProducer;
+
+    public HazelcastClientParams getHazelcast() {
+        return hazelcast;
+    }
+
+    public void setHazelcast(HazelcastClientParams hazelcast) {
+        this.hazelcast = hazelcast;
+    }
+
+    public KafkaConsumerSettings getKafkaConsumer() {
+        return kafkaConsumer;
+    }
+
+    public void setKafkaConsumer(KafkaConsumerSettings kafkaConsumer) {
+        this.kafkaConsumer = kafkaConsumer;
+    }
+
+    public KafkaProducerSettings getKafkaProducer() {
+        return kafkaProducer;
+    }
+
+    public void setKafkaProducer(KafkaProducerSettings kafkaProducer) {
+        this.kafkaProducer = kafkaProducer;
+    }
+}

clearing-parent/csv-importer/src/main/resources/application.properties

@@ -0,0 +1,19 @@
+spring.main.web-application-type=none
+csv-importer.hazelcast.cluster-members=127.0.0.1:5701
+csv-importer.hazelcast.login=dev
+csv-importer.hazelcast.password=dev-pass
+
+csv-importer.kafka-consumer.bootstrap-servers=localhost:9092
+csv-importer.kafka-consumer.group-id=dev-group-csv-importer
+csv-importer.kafka-consumer.enable-auto-commit=true
+csv-importer.kafka-consumer.session-timeout-ms=30000
+csv-importer.kafka-consumer.auto-offset-reset=latest
+csv-importer.kafka-consumer.linger-ms=1
+csv-importer.kafka-consumer.buffer-memory=33554432
+
+csv-importer.kafka-producer.bootstrap-servers=localhost:9092
+csv-importer.kafka-producer.acks=all
+csv-importer.kafka-producer.retries=0
+csv-importer.kafka-producer.batch-size=16384
+csv-importer.kafka-producer.linger-ms=1
+csv-importer.kafka-producer.buffer-memory=33554432

clearing-parent/default-management/src/main/resources/application.properties

@@ -4,7 +4,7 @@ default-management.hazelcast.login=dev
 default-management.hazelcast.password=dev-pass
 
 default-management.kafka-consumer.bootstrap-servers=localhost:9092
-default-management.kafka-consumer.group-id=dev-group-company-service
+default-management.kafka-consumer.group-id=dev-group-default-management
 default-management.kafka-consumer.enable-auto-commit=true
 default-management.kafka-consumer.session-timeout-ms=30000
 default-management.kafka-consumer.auto-offset-reset=latest

clearing-parent/pom.xml

@@ -47,6 +47,7 @@
         <module>xml-importer</module>
         <module>xml-exporter</module>
         <module>default-management</module>
+        <module>csv-importer</module>
     </modules>
 
     <properties>

pom.xml

@@ -57,6 +57,7 @@
         <folder_root_xml-exporter>${folder_root_clearing}/clearing-parent/xml-exporter</folder_root_xml-exporter>
         <folder_root_xml-importer>${folder_root_clearing}/clearing-parent/xml-importer</folder_root_xml-importer>
         <folder_root_df-management>${folder_root_clearing}/clearing-parent/default-management</folder_root_df-management>
+        <folder_root_csv-importer>${folder_root_clearing}/clearing-parent/csv-importer</folder_root_csv-importer>
         <!-- IMDG -->
         <external_libraries.hazelcast.version>3.12.4</external_libraries.hazelcast.version>
         <external_libraries.slf4j.version>1.7.33</external_libraries.slf4j.version>

z-distr/pom.xml

@@ -664,6 +664,29 @@
                                     </fileSets>
                                 </configuration>
                             </execution>
+                            <execution>
+                                <id>copy-csv-importer-bin</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>copy</goal>
+                                </goals>
+                                <configuration>
+                                    <fileSets>
+                                        <fileSet>
+                                            <sourceFile>${folder_root_csv-importer}/target/csv-importer.jar</sourceFile>
+                                            <destinationFile>${folder.clearing.distr.bin}/csv-importer.jar</destinationFile>
+                                        </fileSet>
+                                        <fileSet>
+                                            <sourceFile>${folder_root_csv-importer}/target/csv-importer.jar</sourceFile>
+                                            <destinationFile>${folder.clearing.distr.services}/csv-importer/csv-importer.jar</destinationFile>
+                                        </fileSet>
+                                        <fileSet>
+                                            <sourceFile>${folder_root_csv-importer}/src/main/resources/application.properties</sourceFile>
+                                            <destinationFile>${folder.clearing.distr.settings}/csv-importer/application.properties</destinationFile>
+                                        </fileSet>
+                                    </fileSets>
+                                </configuration>
+                            </execution>
                         </executions>
                     </plugin>
                     <plugin>

z-distr/src/main/resources/distr/sh/csv-importer.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+CLEARING_HOME=/opt/mfd/clearing/
+cd $CLEARING_HOME/bin
+
+CMD="java -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=6033 -jar csv-importer.jar --spring.config.location=$CLEARING_HOME/settings/csv-importer/"
+
+$CMD >/dev/null 2>&1 &
+

z-distr/src/main/resources/distr/sh/kill_all.sh

@@ -23,4 +23,5 @@ kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}')
 kill -9 $(ps -ef | grep java | grep xml-exporter.jar | awk '{print $2}')
 kill -9 $(ps -ef | grep java | grep xml-importer.jar | awk '{print $2}')
 kill -9 $(ps -ef | grep java | grep fix-service.jar | awk '{print $2}')
-kill -9 $(ps -ef | grep java | grep default-management.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep default-management.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep csv-importer.jar | awk '{print $2}')

z-distr/src/main/resources/distr/sh/launch_all.sh

@@ -24,3 +24,4 @@ cd /opt/mfd/clearing/bin
 /opt/mfd/clearing/bin/xml-importer.sh
 /opt/mfd/clearing/bin/fix-service.sh
 /opt/mfd/clearing/bin/default-management.sh
+/opt/mfd/clearing/bin/csv-importer.sh