diff --git a/clearing-parent/csv-importer/pom.xml b/clearing-parent/csv-importer/pom.xml new file mode 100644 index 000000000..61067a70b --- /dev/null +++ b/clearing-parent/csv-importer/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + + ru.spcex.clearing + clearing-parent + SPCEX-3.23.295 + + + csv-importer + + + 17 + 17 + UTF-8 + + + + + ru.spcex.platform + platform-messaging + + + ru.spcex.platform + platform-imdg-api-hazelcast-impl + + + ru.spcex.clearing + security-util + + + ru.spcex.clearing + classes + + + ru.spcex.platform + platform-enum + + + ru.spcex.clearing + clearing-validation + + + org.springframework.boot + spring-boot-starter + + + com.fasterxml.jackson.core + jackson-databind + + + + net.logstash.logback + logstash-logback-encoder + 7.0.1 + + + org.springframework.boot + spring-boot-starter-test + test + + + ru.spcex.clearing + test-clearing + test + + + + + + + src/main/resources + + application.properties + + false + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + + + ${project.artifactId} + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.21.0 + + + org.junit.platform + junit-platform-surefire-provider + 1.2.0-M1 + + + org.junit.jupiter + junit-jupiter-engine + 5.2.0-M1 + + + + + + + \ No newline at end of file diff --git a/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/CsvImporterApplication.java b/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/CsvImporterApplication.java new file mode 100644 index 000000000..01bff0733 --- /dev/null +++ b/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/CsvImporterApplication.java @@ -0,0 +1,18 @@ +package ru.spcex.clearing.csv.importer; + +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; + +@SpringBootApplication +public class CsvImporterApplication { + public static void main(String[] args) { + try { + SpringApplicationBuilder builder = new SpringApplicationBuilder(CsvImporterApplication.class); + builder.run(args); + } catch (Exception e) { + LoggerFactory.getLogger(CsvImporterApplication.class).error("csv-importer start failed: {} -> {}", e.getClass().getSimpleName(), e.getMessage()); + System.exit(-1); + } + } +} diff --git a/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/AppConfig.java b/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/AppConfig.java new file mode 100644 index 000000000..1e8722633 --- /dev/null +++ b/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/AppConfig.java @@ -0,0 +1,47 @@ +package ru.spcex.clearing.csv.importer.config; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import ru.spcex.clearing.csv.importer.config.settings.CsvImporterSettings; +import ru.spcex.platform.imdg.api.ImdgProvider; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; + +@Configuration +public class AppConfig { + + private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) { + ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor(); + if (maxPoolSz > 2) { + pool.setKeepAliveSeconds(60); + pool.setAllowCoreThreadTimeOut(true); + } + pool.setCorePoolSize(maxPoolSz); + pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion); + return pool; + } + + @Bean(name = "taskExecutorHazelcastClientInitializer") + public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() { + return createThreadPoolTaskExecutor(1, true); + } + + @Bean(name = "taskExecutorIdGeneratorAwaiter") + public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() { + return createThreadPoolTaskExecutor(1, false); + } + + @Autowired + @Bean + public ImdgProvider imdgProvider( + @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + CsvImporterSettings settings + ) { + return new HazelcastService(taskExecutorHazelcastClientInitializer, + taskExecutorIdGeneratorAwaiter, + settings.getHazelcast()); + } +} diff --git a/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/settings/CsvImporterSettings.java b/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/settings/CsvImporterSettings.java new file mode 100644 index 000000000..d36f1ce8a --- /dev/null +++ b/clearing-parent/csv-importer/src/main/java/ru/spcex/clearing/csv/importer/config/settings/CsvImporterSettings.java @@ -0,0 +1,41 @@ +package ru.spcex.clearing.csv.importer.config.settings; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; +import ru.spcex.clearing.platform.messaging.config.element.KafkaConsumerSettings; +import ru.spcex.clearing.platform.messaging.config.element.KafkaProducerSettings; +import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams; + +@Component +@PropertySource("file:${spring.config.location}/application.properties") +@ConfigurationProperties("csv-importer") +public class CsvImporterSettings { + private HazelcastClientParams hazelcast; + private KafkaConsumerSettings kafkaConsumer; + private KafkaProducerSettings kafkaProducer; + + public HazelcastClientParams getHazelcast() { + return hazelcast; + } + + public void setHazelcast(HazelcastClientParams hazelcast) { + this.hazelcast = hazelcast; + } + + public KafkaConsumerSettings getKafkaConsumer() { + return kafkaConsumer; + } + + public void setKafkaConsumer(KafkaConsumerSettings kafkaConsumer) { + this.kafkaConsumer = kafkaConsumer; + } + + public KafkaProducerSettings getKafkaProducer() { + return kafkaProducer; + } + + public void setKafkaProducer(KafkaProducerSettings kafkaProducer) { + this.kafkaProducer = kafkaProducer; + } +} diff --git a/clearing-parent/csv-importer/src/main/resources/application.properties b/clearing-parent/csv-importer/src/main/resources/application.properties new file mode 100644 index 000000000..0e658e601 --- /dev/null +++ b/clearing-parent/csv-importer/src/main/resources/application.properties @@ -0,0 +1,19 @@ +spring.main.web-application-type=none +csv-importer.hazelcast.cluster-members=127.0.0.1:5701 +csv-importer.hazelcast.login=dev +csv-importer.hazelcast.password=dev-pass + +csv-importer.kafka-consumer.bootstrap-servers=localhost:9092 +csv-importer.kafka-consumer.group-id=dev-group-csv-importer +csv-importer.kafka-consumer.enable-auto-commit=true +csv-importer.kafka-consumer.session-timeout-ms=30000 +csv-importer.kafka-consumer.auto-offset-reset=latest +csv-importer.kafka-consumer.linger-ms=1 +csv-importer.kafka-consumer.buffer-memory=33554432 + +csv-importer.kafka-producer.bootstrap-servers=localhost:9092 +csv-importer.kafka-producer.acks=all +csv-importer.kafka-producer.retries=0 +csv-importer.kafka-producer.batch-size=16384 +csv-importer.kafka-producer.linger-ms=1 +csv-importer.kafka-producer.buffer-memory=33554432 \ No newline at end of file diff --git a/clearing-parent/default-management/src/main/resources/application.properties b/clearing-parent/default-management/src/main/resources/application.properties index c5e939da0..e306c571e 100644 --- a/clearing-parent/default-management/src/main/resources/application.properties +++ b/clearing-parent/default-management/src/main/resources/application.properties @@ -4,7 +4,7 @@ default-management.hazelcast.login=dev default-management.hazelcast.password=dev-pass default-management.kafka-consumer.bootstrap-servers=localhost:9092 -default-management.kafka-consumer.group-id=dev-group-company-service +default-management.kafka-consumer.group-id=dev-group-default-management default-management.kafka-consumer.enable-auto-commit=true default-management.kafka-consumer.session-timeout-ms=30000 default-management.kafka-consumer.auto-offset-reset=latest diff --git a/clearing-parent/pom.xml b/clearing-parent/pom.xml index 01494a87a..fe8c3d3da 100644 --- a/clearing-parent/pom.xml +++ b/clearing-parent/pom.xml @@ -47,6 +47,7 @@ xml-importer xml-exporter default-management + csv-importer diff --git a/pom.xml b/pom.xml index eec66af06..47fae713d 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,7 @@ ${folder_root_clearing}/clearing-parent/xml-exporter ${folder_root_clearing}/clearing-parent/xml-importer ${folder_root_clearing}/clearing-parent/default-management + ${folder_root_clearing}/clearing-parent/csv-importer 3.12.4 1.7.33 diff --git a/z-distr/pom.xml b/z-distr/pom.xml index 5d7740dad..694bfc4c9 100644 --- a/z-distr/pom.xml +++ b/z-distr/pom.xml @@ -664,6 +664,29 @@ + + copy-csv-importer-bin + prepare-package + + copy + + + + + ${folder_root_csv-importer}/target/csv-importer.jar + ${folder.clearing.distr.bin}/csv-importer.jar + + + ${folder_root_csv-importer}/target/csv-importer.jar + ${folder.clearing.distr.services}/csv-importer/csv-importer.jar + + + ${folder_root_csv-importer}/src/main/resources/application.properties + ${folder.clearing.distr.settings}/csv-importer/application.properties + + + + diff --git a/z-distr/src/main/resources/distr/sh/csv-importer.sh b/z-distr/src/main/resources/distr/sh/csv-importer.sh new file mode 100644 index 000000000..2a78edf16 --- /dev/null +++ b/z-distr/src/main/resources/distr/sh/csv-importer.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +CLEARING_HOME=/opt/mfd/clearing/ +cd $CLEARING_HOME/bin + +CMD="java -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=6033 -jar csv-importer.jar --spring.config.location=$CLEARING_HOME/settings/csv-importer/" + +$CMD >/dev/null 2>&1 & + diff --git a/z-distr/src/main/resources/distr/sh/kill_all.sh b/z-distr/src/main/resources/distr/sh/kill_all.sh index e8137e457..b841bd5f0 100644 --- a/z-distr/src/main/resources/distr/sh/kill_all.sh +++ b/z-distr/src/main/resources/distr/sh/kill_all.sh @@ -23,4 +23,5 @@ kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}') kill -9 $(ps -ef | grep java | grep xml-exporter.jar | awk '{print $2}') kill -9 $(ps -ef | grep java | grep xml-importer.jar | awk '{print $2}') kill -9 $(ps -ef | grep java | grep fix-service.jar | awk '{print $2}') -kill -9 $(ps -ef | grep java | grep default-management.jar | awk '{print $2}') \ No newline at end of file +kill -9 $(ps -ef | grep java | grep default-management.jar | awk '{print $2}') +kill -9 $(ps -ef | grep java | grep csv-importer.jar | awk '{print $2}') \ No newline at end of file diff --git a/z-distr/src/main/resources/distr/sh/launch_all.sh b/z-distr/src/main/resources/distr/sh/launch_all.sh index 89152ed3d..5ca60640d 100644 --- a/z-distr/src/main/resources/distr/sh/launch_all.sh +++ b/z-distr/src/main/resources/distr/sh/launch_all.sh @@ -24,3 +24,4 @@ cd /opt/mfd/clearing/bin /opt/mfd/clearing/bin/xml-importer.sh /opt/mfd/clearing/bin/fix-service.sh /opt/mfd/clearing/bin/default-management.sh +/opt/mfd/clearing/bin/csv-importer.sh