treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add new module csv-importer

Browse source
This commit is contained in:
etreschenkov 2026-04-15 11:31:54 +03:00
parent b7eeb13040
commit e8d2424af5
12 changed files with 279 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

116
clearing-parent/csv-importer/pom.xml Normal file
View file

@ -0,0 +1,116 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ru.spcex.clearing</groupId>
<artifactId>clearing-parent</artifactId>
<version>SPCEX-3.23.295</version>
</parent>
<artifactId>csv-importer</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-messaging</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-imdg-api-hazelcast-impl</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>security-util</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>classes</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.platform</groupId>
<artifactId>platform-enum</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>clearing-validation</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<!-- special logging -->
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>7.0.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>test-clearing</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>application.properties</exclude>
</excludes>
<filtering>false</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
<configuration>
<finalName>${project.artifactId}</finalName>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.21.0</version>
<dependencies>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-surefire-provider</artifactId>
<version>1.2.0-M1</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.2.0-M1</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
</project>

18
clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/CsvImporterApplication.java Normal file
View file

@ -0,0 +1,18 @@
package ru.spcex.clearing.csv.importer;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
@SpringBootApplication
public class CsvImporterApplication {
public static void main(String[] args) {
try {
SpringApplicationBuilder builder = new SpringApplicationBuilder(CsvImporterApplication.class);
builder.run(args);
} catch (Exception e) {
LoggerFactory.getLogger(CsvImporterApplication.class).error("csv-importer start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage());
System.exit(-1);
}
}
}

47
clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/AppConfig.java Normal file
View file

@ -0,0 +1,47 @@
package ru.spcex.clearing.csv.importer.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import ru.spcex.clearing.csv.importer.config.settings.CsvImporterSettings;
import ru.spcex.platform.imdg.api.ImdgProvider;
import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService;
@Configuration
public class AppConfig {
private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) {
ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
if (maxPoolSz > 2) {
pool.setKeepAliveSeconds(60);
pool.setAllowCoreThreadTimeOut(true);
}
pool.setCorePoolSize(maxPoolSz);
pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion);
return pool;
}
@Bean(name = "taskExecutorHazelcastClientInitializer")
public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() {
return createThreadPoolTaskExecutor(1, true);
}
@Bean(name = "taskExecutorIdGeneratorAwaiter")
public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() {
return createThreadPoolTaskExecutor(1, false);
}
@Autowired
@Bean
public ImdgProvider imdgProvider(
@Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
@Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
CsvImporterSettings settings
) {
return new HazelcastService(taskExecutorHazelcastClientInitializer,
taskExecutorIdGeneratorAwaiter,
settings.getHazelcast());
}
}

41
clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/settings/CsvImporterSettings.java Normal file
View file

@ -0,0 +1,41 @@
package ru.spcex.clearing.csv.importer.config.settings;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.platform.messaging.config.element.KafkaConsumerSettings;
import ru.spcex.clearing.platform.messaging.config.element.KafkaProducerSettings;
import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams;
@Component
@PropertySource("file:${spring.config.location}/application.properties")
@ConfigurationProperties("csv-importer")
public class CsvImporterSettings {
private HazelcastClientParams hazelcast;
private KafkaConsumerSettings kafkaConsumer;
private KafkaProducerSettings kafkaProducer;
public HazelcastClientParams getHazelcast() {
return hazelcast;
}
public void setHazelcast(HazelcastClientParams hazelcast) {
this.hazelcast = hazelcast;
}
public KafkaConsumerSettings getKafkaConsumer() {
return kafkaConsumer;
}
public void setKafkaConsumer(KafkaConsumerSettings kafkaConsumer) {
this.kafkaConsumer = kafkaConsumer;
}
public KafkaProducerSettings getKafkaProducer() {
return kafkaProducer;
}
public void setKafkaProducer(KafkaProducerSettings kafkaProducer) {
this.kafkaProducer = kafkaProducer;
}
}

19
clearing-parent/csv-importer/src/main/resources/application.properties Normal file
View file

@ -0,0 +1,19 @@
spring.main.web-application-type=none
csv-importer.hazelcast.cluster-members=127.0.0.1:5701
csv-importer.hazelcast.login=dev
csv-importer.hazelcast.password=dev-pass
csv-importer.kafka-consumer.bootstrap-servers=localhost:9092
csv-importer.kafka-consumer.group-id=dev-group-csv-importer
csv-importer.kafka-consumer.enable-auto-commit=true
csv-importer.kafka-consumer.session-timeout-ms=30000
csv-importer.kafka-consumer.auto-offset-reset=latest
csv-importer.kafka-consumer.linger-ms=1
csv-importer.kafka-consumer.buffer-memory=33554432
csv-importer.kafka-producer.bootstrap-servers=localhost:9092
csv-importer.kafka-producer.acks=all
csv-importer.kafka-producer.retries=0
csv-importer.kafka-producer.batch-size=16384
csv-importer.kafka-producer.linger-ms=1
csv-importer.kafka-producer.buffer-memory=33554432

2
clearing-parent/default-management/src/main/resources/application.properties
View file

@ -4,7 +4,7 @@ default-management.hazelcast.login=dev
default-management.hazelcast.password=dev-pass
default-management.kafka-consumer.bootstrap-servers=localhost:9092
default-management.kafka-consumer.group-id=dev-group-company-service
default-management.kafka-consumer.group-id=dev-group-default-management
default-management.kafka-consumer.enable-auto-commit=true
default-management.kafka-consumer.session-timeout-ms=30000
default-management.kafka-consumer.auto-offset-reset=latest

1
clearing-parent/pom.xml
View file

@ -47,6 +47,7 @@
<module>xml-importer</module>
<module>xml-exporter</module>
<module>default-management</module>
<module>csv-importer</module>
</modules>
<properties>

1
pom.xml
View file

@ -57,6 +57,7 @@
<folder_root_xml-exporter>${folder_root_clearing}/clearing-parent/xml-exporter</folder_root_xml-exporter>
<folder_root_xml-importer>${folder_root_clearing}/clearing-parent/xml-importer</folder_root_xml-importer>
<folder_root_df-management>${folder_root_clearing}/clearing-parent/default-management</folder_root_df-management>
<folder_root_csv-importer>${folder_root_clearing}/clearing-parent/csv-importer</folder_root_csv-importer>
<!-- IMDG -->
<external_libraries.hazelcast.version>3.12.4</external_libraries.hazelcast.version>
<external_libraries.slf4j.version>1.7.33</external_libraries.slf4j.version>

23
z-distr/pom.xml
View file

@ -664,6 +664,29 @@
</fileSets>
</configuration>
</execution>
<execution>
<id>copy-csv-importer-bin</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<fileSets>
<fileSet>
<sourceFile>${folder_root_csv-importer}/target/csv-importer.jar</sourceFile>
<destinationFile>${folder.clearing.distr.bin}/csv-importer.jar</destinationFile>
</fileSet>
<fileSet>
<sourceFile>${folder_root_csv-importer}/target/csv-importer.jar</sourceFile>
<destinationFile>${folder.clearing.distr.services}/csv-importer/csv-importer.jar</destinationFile>
</fileSet>
<fileSet>
<sourceFile>${folder_root_csv-importer}/src/main/resources/application.properties</sourceFile>
<destinationFile>${folder.clearing.distr.settings}/csv-importer/application.properties</destinationFile>
</fileSet>
</fileSets>
</configuration>
</execution>
</executions>
</plugin>
<plugin>

9
z-distr/src/main/resources/distr/sh/csv-importer.sh Normal file
View file

@ -0,0 +1,9 @@
#!/bin/bash
CLEARING_HOME=/opt/mfd/clearing/
cd $CLEARING_HOME/bin
CMD="java -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=6033 -jar csv-importer.jar --spring.config.location=$CLEARING_HOME/settings/csv-importer/"
$CMD >/dev/null 2>&1 &

3
z-distr/src/main/resources/distr/sh/kill_all.sh
View file

@ -23,4 +23,5 @@ kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep xml-exporter.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep xml-importer.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep fix-service.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep default-management.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep default-management.jar | awk '{print $2}')
kill -9 $(ps -ef | grep java | grep csv-importer.jar | awk '{print $2}')

1
z-distr/src/main/resources/distr/sh/launch_all.sh
View file

@ -24,3 +24,4 @@ cd /opt/mfd/clearing/bin
/opt/mfd/clearing/bin/xml-importer.sh
/opt/mfd/clearing/bin/fix-service.sh
/opt/mfd/clearing/bin/default-management.sh
/opt/mfd/clearing/bin/csv-importer.sh