Merge remote-tracking branch 'refs/remotes/origin/dev_gitlab_cicd' into dev

.gitlab-ci.yml

@@ -0,0 +1,54 @@
+image:
+  name: ${DOCKER_PROXY}/docker:latest
+  pull_policy: if-not-present
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  DOCKER_PROXY: "mirror.gcr.io"
+  ACCESS_TOKEN: "glpat-t_i_xh-qo58ud2hT5vxj"
+
+stages:
+  - preflight
+  - build
+  - test
+  - collect_results
+
+preflight:
+  image: ${DOCKER_PROXY}/bitnami/git:latest
+  stage: preflight
+  script:
+    - pwd
+    - git clone -b dev "http://gitlab-ci-token:$ACCESS_TOKEN@git.mfd.msk/mfd/clearing_test.git"
+  artifacts:
+    name: "$CI_COMMIT_REF_NAME"
+    expire_in: 1 day
+    paths:
+      - clearing_test
+
+build:
+  image: ${DOCKER_PROXY}/maven:latest
+  stage: build
+  script:
+    - mvn clean package -DskipTests -Pdistributive
+  artifacts:
+    name: "$CI_COMMIT_REF_NAME"
+    expire_in: 1 day
+    paths:
+      - /builds/mfd/clearing/z-distr/target/clearing
+
+test:
+  stage: test
+  variables:
+    CLEARING_TEST_DIR: "/builds/mfd/clearing/clearing_test"
+    CLR_TEST_DIR: "/builds/mfd/clearing/clr_test"
+    RESULT_FILES_DIR: "/builds/mfd/clearing/clr_test/result-files"
+  script:
+    - echo "Installing maven and python" && apk update && apk add maven python3
+    - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin
+    - echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./db
+    - python3 clr_test/run_test_docker_containers.py
+  artifacts:
+    expire_in: 1 day
+    when: always
+    paths:
+      - $CI_PROJECT_DIR/clr_test/mnt

clr_test/clearing-all/Dockerfile

@@ -0,0 +1,43 @@
+FROM mirror.gcr.io/openjdk:17-slim
+
+ENV CLEARING_HOME="/opt/clearing"
+ENV CLEARING_HOME_BIN="/opt/clearing/bin"
+ENV CLEARING_HOME_CFG="/opt/clearing/bin"
+
+ADD ./bin $CLEARING_HOME_BIN
+ADD ./settings $CLEARING_HOME/settings
+
+ADD ./kill_all.sh $CLEARING_HOME/kill_all.sh
+ADD ./launch_all.sh $CLEARING_HOME/launch_all.sh
+ADD ./launch_and_test.sh $CLEARING_HOME/launch_and_test.sh
+
+ADD ./clearing_testing_utils $CLEARING_HOME/clearing_testing_utils
+
+USER root
+
+RUN chmod +x $CLEARING_HOME/*.sh
+RUN chmod +x $CLEARING_HOME/clearing_testing_utils/*.sh
+RUN chmod +x $CLEARING_HOME/clearing_testing_utils/clearing-checker/*.sh
+RUN chmod +x $CLEARING_HOME/clearing_testing_utils/clearing-converter/*.sh
+RUN chmod +x $CLEARING_HOME/clearing_testing_utils/clearing-tester/*.sh
+RUN mkdir -p $CLEARING_HOME/files/swt/importer/in
+RUN mkdir -p $CLEARING_HOME/files/swt/importer/out
+RUN mkdir -p $CLEARING_HOME/files/swt/importer/error
+RUN mkdir -p $CLEARING_HOME/files/swt/exporter
+RUN mkdir -p $CLEARING_HOME/files/dbf/importer/in
+RUN mkdir -p $CLEARING_HOME/files/dbf/importer/out
+RUN mkdir -p $CLEARING_HOME/files/dbf/importer/error
+RUN mkdir -p $CLEARING_HOME/files/dbf/exporter
+RUN mkdir -p $CLEARING_HOME/files/xml/importer/in
+RUN mkdir -p $CLEARING_HOME/files/xml/importer/out
+RUN mkdir -p $CLEARING_HOME/files/xml/importer/error
+RUN mkdir -p $CLEARING_HOME/files/xml/exporter
+
+WORKDIR $CLEARING_HOME
+RUN sed -i 's/\r$//' $CLEARING_HOME/*.sh
+RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/*.sh
+RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-checker/*.sh
+RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-converter/*.sh
+RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-tester/*.sh
+
+ENTRYPOINT [ "./launch_and_test.sh" ]

clr_test/clearing-all/clearing_testing_utils/clearing-checker/application.properties

@@ -0,0 +1,4 @@
+server.port=9902
+
+dbf-swt-checker.encoding=UTF-8
+dbf-swt-checker.rootDir=/opt/clearing/clearing_testing_utils/clearing-checker/res/

clr_test/clearing-all/clearing_testing_utils/clearing-checker/clearing-checker.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
+cd $CLEARING_TEST_HOME/clearing-checker
+
+CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9992  -jar dbf-swt-checker-1.0.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-checker/"
+
+$CMD >/dev/null 2>&1 &

clr_test/clearing-all/clearing_testing_utils/clearing-converter/application.properties

@@ -0,0 +1,4 @@
+clearing-converter.store.root-dir=/opt/clearing/clearing_testing_utils/clearing-converter/
+clearing-converter.encoding-source=cp866
+server.port=9901
+server.servlet.context-path=/converter

clr_test/clearing-all/clearing_testing_utils/clearing-converter/clearing-converter.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
+cd $CLEARING_TEST_HOME/clearing-converter
+
+CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9991  -jar swt-dbf-converter-0.0.1.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-converter/"
+
+$CMD >/dev/null 2>&1 &
+

clr_test/clearing-all/clearing_testing_utils/clearing-tester/README.md

@@ -0,0 +1,32 @@
+Приложение для Тестирования Клиринга
+Обзор
+Приложение для тестирования клиринга - это инструмент для облегчения тестирования и анализа сессий клиринга. Оно предоставляет основу для обработки файлов, управления сессиями и анализа результатов, обеспечивая упрощенный процесс управления операциями клиринга.
+
+Структура
+src/main/java/: Содержит все исходные файлы Java.
+ru/spcex/clearing/tester/: Основной пакет приложения.
+Application.java: Точка входа в приложение.
+config/: Классы конфигурации приложения.
+enums/: Перечисления, используемые в приложении.
+models/: Данные модели для представления различных сущностей.
+service/: Сервисы, предоставляющие основные функциональности.
+src/main/resources/: Содержит файлы конфигурации и другие ресурсы.
+application.properties: Свойства и конфигурация приложения.
+logback.xml: Конфигурация для логирования.
+Предварительные требования
+Java JDK 17 или выше
+Maven
+Установка и настройка
+Клонировать репозиторий: Склонируйте или загрузите исходный код на свой локальный компьютер.
+Установить зависимости: Перейдите в корневую директорию проекта и выполните команду mvn install для установки всех необходимых зависимостей.
+Настройка пропертей: Настройте application.properties и другие файлы конфигурации в соответствии с вашей средой и потребностями.
+Запуск приложения
+Выполните следующую команду из корневого каталога проекта:
+
+java -jar путь/до/clearing-tester.jar --spring.config.location=путь до папки с ресурсами (пример --spring.config.location=D:\repo\mfd\clearing-tester\src\main\resources\)
+
+Замените путь/до/clearing-tester.jar на реальный путь к скомпилированному JAR-файлу.
+
+Также для корректной работы требуется 2 сервиса:
+dbf-swt-converter - для их запуска также требуется ввести java -jar путь/до/clearing-tester.jar --spring.config.location=путь до папки с ресурсами (пример --spring.config.location=D:\repo\mfd\clearing-tester\src\main\resources\) и правильно заполнить property
+dbf-swt-checker

clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties

@@ -0,0 +1,67 @@
+server.port=9900
+#local storage settings
+clearing-tester.store.root-dir=/opt/clearing/clearing_testing_utils/store
+
+#sftp storage settings
+clearing-tester.sftp.dbf.importer.in-dir=/opt/clearing/files/dbf/importer/in
+clearing-tester.sftp.dbf.importer.out-dir=/opt/clearing/files/dbf/importer/out
+clearing-tester.sftp.dbf.exporter.out-dir=/opt/clearing/files/dbf/exporter
+
+clearing-tester.sftp.swt.importer.in-dir=/opt/clearing/files/swt/importer/in
+clearing-tester.sftp.swt.importer.out-dir=/opt/clearing/files/swt/importer/out
+clearing-tester.sftp.swt.exporter.out-dir=/opt/clearing/files/swt/exporter
+
+clearing-tester.sftp.xml.importer.in-dir=/opt/clearing/files/xml/importer/in
+clearing-tester.sftp.xml.importer.out-dir=/opt/clearing/files/xml/importer/out
+clearing-tester.sftp.xml.exporter.out-dir=/opt/clearing/files/xml/exporter
+
+clearing-tester.sftp.user=user
+clearing-tester.sftp.password=Aa111111
+clearing-tester.sftp.server-ip=sftp
+clearing-tester.sftp.server-port=22
+
+#converter service url
+clearing-tester.converter-url=http://localhost:9901/converter/api/convert
+clearing-tester.type-of-conversion=DBF-OK
+# DBF-OK, DBF-!OK, DBF--OK, DBF-EMPTY
+
+#converter service url			
+clearing-tester.checker-url=http://localhost:9902/api/checkFiles
+
+#mock dates settings
+clearing-tester.mock-date-for-sdf=2024-07-26
+clearing-tester.mock-date-for-execution=2024-07-26
+
+#hazelcast settings
+clearing-tester.hazelcast.cluster-members=127.0.0.1:5701
+clearing-tester.hazelcast.login=dev
+clearing-tester.hazelcast.password=dev-pass
+
+#database settings
+clearing-tester.database.url=jdbc:postgresql://db:5432/clearing
+clearing-tester.database.login=clearing
+clearing-tester.database.password=Aa111111
+
+#common settings
+clearing-tester.common.encoding-source=cp866
+clearing-tester.common.insert-batch-size=100
+clearing-tester.common.threads-count=10
+
+#kafka settings
+clearing-tester.kafka-producer.bootstrap-servers=kafka:9092
+clearing-tester.kafka-producer.acks=all
+clearing-tester.kafka-producer.retries=0
+clearing-tester.kafka-producer.batch-size=16384
+clearing-tester.kafka-producer.linger-ms=1
+clearing-tester.kafka-producer.buffer-memory=33554432
+
+clearing-tester.kafka-consumer.bootstrap-servers=kafka:9092
+clearing-tester.kafka-consumer.group-id=dev-group-clearing-service
+clearing-tester.kafka-consumer.enable-auto-commit=false
+clearing-tester.kafka-consumer.session-timeout-ms=30000
+clearing-tester.kafka-consumer.auto-offset-reset=latest
+clearing-tester.kafka-consumer.linger-ms=1
+clearing-tester.kafka-consumer.buffer-memory=33554432
+
+clearing-tester.common.session-type=${session_type:TRDT}
+clearing-tester.common.section=${section:FOND}

clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
+cd $CLEARING_TEST_HOME/clearing-tester
+
+CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ --spring.profiles.active=dev"
+
+$CMD

clr_test/clearing-all/clearing_testing_utils/kill_test_all.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+kill -9 $(ps -ef | grep java | grep clearing2/ | grep dbf-swt-checker-1.0.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep clearing2/ | grep swt-dbf-converter-0.0.1.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep clearing2/ | grep clearing-tester.jar | awk '{print $2}')

clr_test/clearing-all/clearing_testing_utils/restart_test_all.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+sh ./kill_test_all.sh
+sh ./run_test_all.sh

clr_test/clearing-all/clearing_testing_utils/run_test_env.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cd /opt/clearing/clearing_testing_utils
+
+sh ./clearing-checker/clearing-checker.sh
+sh ./clearing-converter/clearing-converter.sh
+
+echo Test finish. xit code: $?

clr_test/clearing-all/kill_all.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+kill -9 $(ps -ef | grep java | grep account-service.jar  | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep backend-api.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep balance-service.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep company-service.jar  | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep clearing-service.jar  | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep dbf-exporter.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep lim-exporter.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep swt-exporter.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep dbf-importer.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep trade-importer.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep imdg.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep securities-service.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep utility-service.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep scheduler-service.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep reports-service.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep registry-service.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep swt-importer.jar | awk '{print $2}')
+kill -9 $(ps -ef | grep java | grep gateway-api.jar | awk '{print $2}')

clr_test/clearing-all/launch_all.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+CLEARING_HOME=/opt/clearing/
+CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
+cd $CLEARING_HOME/bin
+
+echo Start starting all in $CLEARING_HOME.
+
+java -jar $CLEARING_HOME/bin/imdg.jar --spring.config.location=$CLEARING_HOME/settings/imdg/ &
+java -jar $CLEARING_HOME/bin/backend-api.jar --spring.config.location=$CLEARING_HOME/settings/backend-api/ &
+#java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEARING_HOME/settings/account-service/  &
+#java -jar $CLEARING_HOME/bin/balance-service.jar --spring.config.location=$CLEARING_HOME/settings/balance-service/   &
+java -jar $CLEARING_HOME/bin/clearing-service.jar --spring.config.location=$CLEARING_HOME/settings/clearing-service/ &
+java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test &
+#java -jar $CLEARING_HOME/bin/company-service.jar --spring.config.location=$CLEARING_HOME/settings/company-service/   &
+java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test &
+#java -jar $CLEARING_HOME/bin/gateway-api.jar --spring.config.location=$CLEARING_HOME/settings/gateway-api/   &
+#java -jar $CLEARING_HOME/bin/registry-service.jar --spring.config.location=$CLEARING_HOME/settings/registry-service/  &
+#java -jar $CLEARING_HOME/bin/reports-service.jar --spring.config.location=$CLEARING_HOME/settings/reports-service/  &
+java -jar $CLEARING_HOME/bin/scheduler-service.jar --spring.config.location=$CLEARING_HOME/settings/scheduler-service/ &
+#java -jar $CLEARING_HOME/bin/securities-service.jar --spring.config.location=$CLEARING_HOME/settings/securities-service/  &
+java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ --spring.profiles.active=test &
+java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ --spring.profiles.active=test &
+#java -jar $CLEARING_TEST_HOME/clearing-tester/clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ &
+echo all services launched starting. $?

clr_test/clearing-all/launch_and_test.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+CLEARING_HOME=/opt/clearing/
+cd $CLEARING_HOME/
+
+date
+
+echo Start CLEARING system
+sh ./launch_all.sh
+sleep 180
+
+echo Run test case
+sh ./clearing_testing_utils/run_test_env.sh
+sleep 30
+
+echo Run clearing-tester
+sh ./clearing_testing_utils/clearing-tester/clearing-tester.sh
+
+if [ $? -eq 0 ]
+then
+  echo "Tests passed successfully"
+  exit 0
+else
+  echo "Tests failed"
+  exit 1
+fi

clr_test/clearing-all/settings/backend-api/application.properties

@@ -0,0 +1,28 @@
+server.port=8080
+server.servlet.context-path=/backend-api
+server.ssl.key-store-type=PKCS12
+server.ssl.key-store=classpath:keystore/client.p12
+server.ssl.key-store-password=********
+server.ssl.enabled=false
+spring.main.web-application-type=servlet
+backend-api.security.authorization-disabled=true
+backend-api.example-setting=test
+backend-api.hazelcast.cluster-members=127.0.0.1:5701
+backend-api.hazelcast.login=dev
+backend-api.hazelcast.password=dev-pass
+#backend-api.hazelcast-search.cluster-members=127.0.0.1:5702
+#backend-api.hazelcast-search.login=dev-hist
+#backend-api.hazelcast-search.password=dev-pass-hist
+backend-api.kafka-producer.bootstrap-servers=kafka:9092
+backend-api.kafka-producer.acks=all
+backend-api.kafka-producer.retries=0
+backend-api.kafka-producer.batch-size=16384
+backend-api.kafka-producer.linger-ms=1
+backend-api.kafka-producer.buffer-memory=33554432
+backend-api.kafka-consumer.bootstrap-servers=kafka:9092
+backend-api.kafka-consumer.group-id=dev-group-backend-api
+backend-api.kafka-consumer.enable-auto-commit=true
+backend-api.kafka-consumer.session-timeout-ms=30000
+backend-api.kafka-consumer.auto-offset-reset=latest
+backend-api.kafka-consumer.linger-ms=1
+backend-api.kafka-consumer.buffer-memory=33554432

clr_test/clearing-all/settings/backend-api/keycloak.json

@@ -0,0 +1,29 @@
+{
+  "realm" : "master",
+  "resource" : "clearing",
+  "auth-server-url" : "http://10.200.200.147:8080",
+  "ssl-required" : "none",
+  "use-resource-role-mappings" : false,
+  "enable-cors" : true,
+  "cors-max-age" : 1000,
+  "cors-allowed-methods" : "POST, PUT, DELETE, GET",
+  "cors-exposed-headers" : "WWW-Authenticate",
+  "bearer-only" : true,
+  "enable-basic-auth" : false,
+  "expose-token" : false,
+  "verify-token-audience" : false,
+  "credentials" : {
+    "secret" : "WnOVCxcAmrUYc8IjiFHOuRif5Oesoyfr"
+  },
+
+  "connection-pool-size" : 20,
+  "socket-timeout-millis" : 5000,
+  "connection-timeout-millis" : 6000,
+  "connection-ttl-millis" : 500,
+  "disable-trust-manager" : false,
+  "allow-any-hostname" : false,
+
+  "token-minimum-time-to-live" : 0,
+  "min-time-between-jwks-requests" : 10,
+  "public-key-cache-ttl" : 86400
+}

clr_test/clearing-all/settings/clearing-service/application.properties

@@ -0,0 +1,24 @@
+spring.main.web-application-type=none
+
+clearing-service.hazelcast.cluster-members=127.0.0.1:5701
+clearing-service.hazelcast.login=dev
+clearing-service.hazelcast.password=dev-pass
+
+clearing-service.kafka-consumer.bootstrap-servers=kafka:9092
+clearing-service.kafka-consumer.group-id=dev-group-clearing-service
+clearing-service.kafka-consumer.enable-auto-commit=false
+clearing-service.kafka-consumer.session-timeout-ms=30000
+clearing-service.kafka-consumer.auto-offset-reset=latest
+clearing-service.kafka-consumer.linger-ms=1
+clearing-service.kafka-consumer.buffer-memory=33554432
+
+clearing-service.kafka-producer.bootstrap-servers=kafka:9092
+clearing-service.kafka-producer.acks=all
+clearing-service.kafka-producer.retries=0
+clearing-service.kafka-producer.batch-size=16384
+clearing-service.kafka-producer.linger-ms=1
+clearing-service.kafka-producer.buffer-memory=33554432
+
+clearing-service.session-stage.inspection-gateway-timeout=60
+
+clearing-service.scheduler.check-payment-instruction=*/5 * * * * *

clr_test/clearing-all/settings/dbf-exporter/application.properties

@@ -0,0 +1,33 @@
+spring.main.web-application-type=none
+
+export-dbf-service.hazelcast.cluster-members=localhost:5701
+export-dbf-service.hazelcast.login=dev
+export-dbf-service.hazelcast.password=dev-pass
+
+export-dbf-service.common.encoding=cp866
+export-dbf-service.common.threads-count=10
+
+export-dbf-service.store.local-temp-dir=/opt/clearing/files/dbf/exporter/
+# код валюты должен быть в верхнем регистре, например для рублей - "RUB" (*.RUB=export/rub/).RUB=export/rub/
+export-dbf-service.store.out-pay-val-dir.RUB=/opt/clearing/files/dbf/exporter/
+export-dbf-service.store.out-pay-val-dir.EUR=/opt/clearing/files/dbf/exporter/
+export-dbf-service.store.out-pay-val-dir.CNY=/opt/clearing/files/dbf/exporter/
+#export-dbf-service.store.user:tester
+#export-dbf-service.store.password=password
+#export-dbf-service.store.server-ip=10.230.238.53
+#export-dbf-service.store.server-port=2222
+
+export-dbf-service.kafka-consumer.bootstrap-servers=kafka:9092
+export-dbf-service.kafka-consumer.group-id=dev-group-balance-service
+export-dbf-service.kafka-consumer.enable-auto-commit=false
+export-dbf-service.kafka-consumer.session-timeout-ms=30000
+export-dbf-service.kafka-consumer.auto-offset-reset=latest
+export-dbf-service.kafka-consumer.linger-ms=1
+export-dbf-service.kafka-consumer.buffer-memory=33554432
+
+export-dbf-service.kafka-producer.bootstrap-servers=kafka:9092
+export-dbf-service.kafka-producer.acks=all
+export-dbf-service.kafka-producer.retries=0
+export-dbf-service.kafka-producer.batch-size=16384
+export-dbf-service.kafka-producer.linger-ms=1
+export-dbf-service.kafka-producer.buffer-memory=33554432

clr_test/clearing-all/settings/dbf-importer/application.properties

@@ -0,0 +1,41 @@
+spring.main.web-application-type=none
+
+import-dbf-service.scheduler.check-src-dir-cron=*/3 * * * * *
+
+import-dbf-service.store.delete-src-files=false
+import-dbf-service.store.src-dir=/opt/clearing/files/dbf/importer/in
+import-dbf-service.store.out-dir=/opt/clearing/files/dbf/importer/out/
+import-dbf-service.store.out-dir-error=/opt/clearing/files/dbf/importer/error/
+
+import-dbf-service.common.encoding-source=cp866
+import-dbf-service.common.insert-batch-size=100
+import-dbf-service.common.threads-count=10
+
+#sftpSrcDir
+import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=clearing_importer_sftp/rub/
+import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=clearing_importer_sftp/eur/
+import-dbf-service.store.sftp-in.sftp-src-dir=clearing_importer_sftp
+import-dbf-service.store.sftp-in.user=user
+import-dbf-service.store.sftp-in.password=Aa111111
+import-dbf-service.store.sftp-in.server-ip=sftp
+import-dbf-service.store.sftp-in.server-port=22
+
+import-dbf-service.hazelcast.cluster-members=localhost:5701
+import-dbf-service.hazelcast.login=dev
+import-dbf-service.hazelcast.password=dev-pass
+
+
+import-dbf-service.kafka-producer.bootstrap-servers=kafka:9092
+import-dbf-service.kafka-producer.acks=all
+import-dbf-service.kafka-producer.retries=0
+import-dbf-service.kafka-producer.batch-size=16384
+import-dbf-service.kafka-producer.linger-ms=1
+import-dbf-service.kafka-producer.buffer-memory=33554432
+
+import-dbf-service.kafka-consumer.bootstrap-servers=kafka:9092
+import-dbf-service.kafka-consumer.group-id=dev-group-clearing-service
+import-dbf-service.kafka-consumer.enable-auto-commit=false
+import-dbf-service.kafka-consumer.session-timeout-ms=30000
+import-dbf-service.kafka-consumer.auto-offset-reset=latest
+import-dbf-service.kafka-consumer.linger-ms=1
+import-dbf-service.kafka-consumer.buffer-memory=33554432

clr_test/clearing-all/settings/imdg/application.properties

@@ -0,0 +1,13 @@
+imdg.hazelcast.listenPort=5701
+imdg.hazelcast.login=dev
+imdg.hazelcast.password=dev-pass
+imdg.hazelcast.cluster-members[0]=127.0.0.1
+imdg.hazelcast.backup-count=3
+imdg.hazelcast.instance-name=instance
+imdg.hazelcast.db-sync-seconds=10
+imdg.database.login=clearing
+imdg.database.password=Aa111111
+imdg.database.url=jdbc:postgresql://db:5432/clearing
+#imdg.database.url=jdbc:postgresql://10.200.200.133:5432/postgres?currentSchema=clearing_tester
+imdg.database.min-pool-size=10
+imdg.database.max-pool-size=30

clr_test/clearing-all/settings/registry-service/application.properties

@@ -0,0 +1,29 @@
+spring.main.web-application-type=none
+
+registry-service.hazelcast.cluster-members=127.0.0.1:5701
+registry-service.hazelcast.login=dev
+registry-service.hazelcast.password=dev-pass
+
+registry-service.kafka-consumer.bootstrap-servers=kafka:9092
+registry-service.kafka-consumer.group-id=dev-group-registry-service
+registry-service.kafka-consumer.enable-auto-commit=true
+registry-service.kafka-consumer.session-timeout-ms=30000
+registry-service.kafka-consumer.auto-offset-reset=latest
+registry-service.kafka-consumer.linger-ms=1
+registry-service.kafka-consumer.buffer-memory=33554432
+
+registry-service.kafka-producer.bootstrap-servers=kafka:9092
+registry-service.kafka-producer.acks=all
+registry-service.kafka-producer.retries=0
+registry-service.kafka-producer.batch-size=16384
+registry-service.kafka-producer.linger-ms=1
+registry-service.kafka-producer.buffer-memory=33554432
+
+registry-service.csv-mode=true
+registry-service.reports-store.local-temp-dir=./reports_out_temp
+registry-service.reports-store.out-dir=reports
+registry-service.reports-store.user=tester
+registry-service.reports-store.password=password
+registry-service.reports-store.server-ip=10.230.238.53
+registry-service.reports-store.server-port=2222
+registry-service.reports-store.delete-after-send=true

clr_test/clearing-all/settings/scheduler-service/application.properties

@@ -0,0 +1,17 @@
+spring.main.web-application-type=none
+scheduler-service.hazelcast.cluster-members=127.0.0.1:5701
+scheduler-service.hazelcast.login=dev
+scheduler-service.hazelcast.password=dev-pass
+scheduler-service.kafka-consumer.bootstrap-servers=kafka:9092
+scheduler-service.kafka-consumer.group-id=dev-group-scheduler-service
+scheduler-service.kafka-consumer.enable-auto-commit=true
+scheduler-service.kafka-consumer.session-timeout-ms=30000
+scheduler-service.kafka-consumer.auto-offset-reset=latest
+scheduler-service.kafka-consumer.linger-ms=1
+scheduler-service.kafka-consumer.buffer-memory=33554432
+scheduler-service.kafka-producer.bootstrap-servers=kafka:9092
+scheduler-service.kafka-producer.acks=all
+scheduler-service.kafka-producer.retries=0
+scheduler-service.kafka-producer.batch-size=16384
+scheduler-service.kafka-producer.linger-ms=1
+scheduler-service.kafka-producer.buffer-memory=33554432

clr_test/clearing-all/settings/swt-exporter/application.properties

@@ -0,0 +1,31 @@
+spring.main.web-application-type=none
+
+export-swt-service.hazelcast.cluster-members=localhost:5701
+export-swt-service.hazelcast.login=dev
+export-swt-service.hazelcast.password=dev-pass
+
+export-swt-service.common.encoding=cp866
+export-swt-service.common.threads-count=10
+
+export-swt-service.docOut=/opt/clearing/files/swt/exporter/
+
+export-swt-service.kafka-consumer.bootstrap-servers=kafka:9092
+export-swt-service.kafka-consumer.group-id=dev-group-balance-service
+export-swt-service.kafka-consumer.enable-auto-commit=false
+export-swt-service.kafka-consumer.session-timeout-ms=30000
+export-swt-service.kafka-consumer.auto-offset-reset=latest
+export-swt-service.kafka-consumer.linger-ms=1
+export-swt-service.kafka-consumer.buffer-memory=33554432
+
+export-swt-service.kafka-producer.bootstrap-servers=kafka:9092
+export-swt-service.kafka-producer.acks=all
+export-swt-service.kafka-producer.retries=0
+export-swt-service.kafka-producer.batch-size=16384
+export-swt-service.kafka-producer.linger-ms=1
+export-swt-service.kafka-producer.buffer-memory=33554432
+
+export-swt-service.sftp-out.sftp-out-dir=swt_out_sftp_directory
+export-swt-service.sftp-out.user=user
+export-swt-service.sftp-out.password=********
+export-swt-service.sftp-out.server-ip=127.0.0.1
+export-swt-service.sftp-out.server-port=22

clr_test/clearing-all/settings/swt-importer/application.properties

@@ -0,0 +1,41 @@
+server.port=7117
+server.servlet.context-path=/importer-swt
+spring.main.web-application-type=servlet
+
+import-swt-service.scheduler.check-src-dir-cron= */10 * * * * *
+
+import-swt-service.store.delete-src-files=false
+import-swt-service.store.src-dir=/opt/clearing/files/swt/importer/in
+import-swt-service.store.out-dir=/opt/clearing/files/swt/importer/out/
+import-swt-service.store.out-dir-error=/opt/clearing/files/swt/importer/error/
+
+#sftpSrcDir
+import-swt-service.store.sftp-in.sftp-src-dir=/opt/clearing/swt/in
+import-swt-service.store.sftp-in.user=user
+import-swt-service.store.sftp-in.password=*********
+import-swt-service.store.sftp-in.server-ip=127.0.0.1
+import-swt-service.store.sftp-in.server-port=22
+
+import-swt-service.common.encoding-source=cp866
+import-swt-service.common.insert-batch-size=100
+import-swt-service.common.threads-count=10
+
+import-swt-service.hazelcast.cluster-members=localhost:5701
+import-swt-service.hazelcast.login=dev
+import-swt-service.hazelcast.password=dev-pass
+
+
+import-swt-service.kafka-producer.bootstrap-servers=kafka:9092
+import-swt-service.kafka-producer.acks=all
+import-swt-service.kafka-producer.retries=0
+import-swt-service.kafka-producer.batch-size=16384
+import-swt-service.kafka-producer.linger-ms=1
+import-swt-service.kafka-producer.buffer-memory=33554432
+
+import-swt-service.kafka-consumer.bootstrap-servers=kafka:9092
+import-swt-service.kafka-consumer.group-id=dev-group-clearing-service
+import-swt-service.kafka-consumer.enable-auto-commit=false
+import-swt-service.kafka-consumer.session-timeout-ms=30000
+import-swt-service.kafka-consumer.auto-offset-reset=latest
+import-swt-service.kafka-consumer.linger-ms=1
+import-swt-service.kafka-consumer.buffer-memory=33554432

clr_test/db/Dockerfile

@@ -0,0 +1,5 @@
+FROM mirror.gcr.io/postgres:14.1-alpine
+
+ADD ./DDL.sql /docker-entrypoint-initdb.d/
+ADD export.sql /docker-entrypoint-initdb.d/
+ADD test-data.sql /docker-entrypoint-initdb.d/

clr_test/db/test-data.sql

@@ -0,0 +1,8 @@
+INSERT INTO public.s_trades (id,created_at,updated_at,trade_num,operation,class_code,trade_date,sec_code,accruedint,accruedint2,lower_discount,order_num,price,price2,repo_rate,repo_value,repo2_value,start_discount,ts_commission,upper_discount,value,yield,qty,qty_pcs,trade_date_time,repo_term,clearing_commission,exchange_commission,tech_center_commission,account,broker_ref,client_code,settle_code,user_id,exchange_code,firm_id,firm_name,cp_firm_id,cp_firm_name,class_name,sec_name,settle_date,settle_currency,trade_currency,trade_time_ms,bank_acc_id,"section") VALUES
+         (5557910001,'2024-04-16 11:13:30.198','2024-04-16 11:13:30.198',1347782,'S','UVAV','2024-04-16','CNYRUB_TOD_C',0.000000000000000000,0.000000000000000000,0.000000000000000000,874332,13.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,1300000.000000000000000000,0.000000000000000000,100.000000000000000000,100000.000000000000000000,'2024-04-16 17:14:00.000',0,0.000000000000000000,0.000000000000000000,0.000000000000000000,'0079MAT00001','0079MAT00001/','0079MAT00001','T0',NULL,NULL,'79','ГПБ','8','ПАО Сбербанк','СПВБ: Двусторонний непрерывный аукцион (TOD)',NULL,'2024-04-16','RUB',NULL,43707,NULL,'CURR'),
+         (5557910000,'2024-04-16 11:13:30.198','2024-04-16 11:13:30.198',1347782,'B','UVAV','2024-04-16','CNYRUB_TOD_C',0.000000000000000000,0.000000000000000000,0.000000000000000000,874331,13.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,0.000000000000000000,1300000.000000000000000000,0.000000000000000000,100.000000000000000000,100000.000000000000000000,'2024-04-16 17:14:00.000',0,0.000000000000000000,0.000000000000000000,0.000000000000000000,'0008CAT00001','0008CAT00001/','0008CAT00001','T0',NULL,NULL,'8','ПАО Сбербанк','79','ГПБ','СПВБ: Двусторонний непрерывный аукцион (TOD)',NULL,'2024-04-16','RUB',NULL,43707,NULL,'CURR');
+
+INSERT INTO public.execution_currency (exchange_execution_id,exchange_execution_time,exchange_execution_microseconds,trading_date,settlement_date,settlement_code,security_id,security_symbol,security_name,company_id,party_trading_clearing_registry_id,party_trading_clearing_registry,counter_party_id,counter_party_trading_clearing_registry_id,counter_party_trading_clearing_registry,market,price,lots,settlement_amount,quantity,side,currency_code,settlement_organization,coverage_status,session_id,clearing_date,id,created_at,updated_at) VALUES
+         (1347782,'2024-04-16 17:14:00','2024-04-16 17:14:00','2024-04-16','2024-04-16','T0',553161143,'CNYRUB_TOD_C','Валютная пара рубль-юань',2000302,408160009,'0008CAT00001',2000096,5553940004,'0079MAT00001','UVAV',13.000000000000000000,100.00,1300000.00,100000.00,'B','RUB',NULL,NULL,NULL,'2024-04-16',5557920002,'2024-04-16 11:13:31.071',NULL),
+         (1347782,'2024-04-16 17:14:00','2024-04-16 17:14:00','2024-04-16','2024-04-16','T0',553161143,'CNYRUB_TOD_C','Валютная пара рубль-юань',2000096,5553940004,'0079MAT00001',2000302,408160009,'0008CAT00001','UVAV',13.000000000000000000,100.00,1300000.00,100000.00,'S','RUB',NULL,NULL,NULL,'2024-04-16',5557920000,'2024-04-16 11:13:30.86',NULL);
+

clr_test/docker-compose.yml

@@ -0,0 +1,97 @@
+version: "3"
+services:
+  db:
+    build: db
+    restart: always
+    environment:
+      - POSTGRES_USER=clearing
+      - POSTGRES_PASSWORD=Aa111111
+      - POSTGRES_DB=clearing
+      - POSTGRES_HOST_AUTH_METHOD=trust
+    networks:
+      - clearing-net
+    ports:
+      - "5433:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U clearing -h db -p 5432 -d clearing"]
+      interval: 5s
+      timeout: 5s
+      retries: 100
+
+  clearing_system:
+    build: ./clearing-all
+    depends_on:
+      db:
+        condition: service_healthy
+      kafka:
+        condition: service_started
+    networks:
+      - clearing-net
+    ports:
+      - "7010:7010"
+      - "5701:5701"
+      - "7090:7090"
+      - "7020:7020"
+      - "8080:8080"
+    volumes:
+      - ./mnt/results:/opt/clearing/clearing_testing_utils/store/results
+      - ./mnt/logs/clearing:/opt/clearing/bin/log
+      - ./mnt/logs/tester:/opt/clearing/clearing_testing_utils/clearing-tester/logs
+      - ./mnt/logs/converter:/opt/clearing/clearing_testing_utils/clearing-converter/logs
+    # restart: always
+
+  zookeeper:
+    image: mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64
+    container_name: zookeeper
+    networks:
+      - clearing-net
+    environment:
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - "22181:2181"
+
+  kafka:
+    image: mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64
+    container_name: kafka
+    networks:
+      - clearing-net
+    ports:
+      - "19092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:19092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+    depends_on:
+      - zookeeper
+
+  sftp:
+    image: mirror.gcr.io/atmoz/sftp
+    networks:
+      - clearing-net
+    volumes:
+      - /opt/mfd/clr_test/upload:/home/user/upload
+    ports:
+      - "2222:22"
+    command: user:Aa111111:1001
+
+volumes:
+  db:
+    driver: local
+  zookeeper_data:
+    driver: local
+  kafka_data:
+    driver: local
+
+networks:
+  clearing-net:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 192.168.220.0/24

clr_test/run_test_docker_containers.py

@@ -0,0 +1,107 @@
+import os
+import re
+import sys
+import time
+import shutil
+import subprocess
+import logging
+
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+
+
+DATA_ROOT_DIR = os.environ['CLEARING_TEST_DIR']
+CLR_TEST_DIR = os.environ['CLR_TEST_DIR']
+RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
+
+
+def run_docker(folder_name):
+    logging.info(f"Running docker-compose for {folder_name}")
+    os.chdir("clr_test")
+    subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
+    subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
+    subprocess.call('docker rmi -f clr_test-db', shell=True)
+    subprocess.call('docker rmi -f clr_test-clearing_system', shell=True)
+    code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system', shell=True)
+    logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results"))
+    os.chdir("..")
+
+    if code != 0:
+        sys.exit(code)
+
+    return code
+
+
+def change_date(folder_name):
+    pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)"
+
+    with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f:
+        date = f.read()
+    with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'r') as f:
+        file = f.read()
+    re.sub(pattern, date, file)
+    with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'w') as f:
+        f.write(file)
+
+
+def remove_files_with_extensions(extension):
+    for root, dirs, files in os.walk(CLR_TEST_DIR):
+        for currentFile in files:
+            if currentFile.lower().endswith(extension):
+                logging.info(f"Removing file {os.path.join(root, currentFile)}, based on extension {extension}")
+                os.remove(os.path.join(root, currentFile))
+
+
+def run_specific(index):
+    logging.info(f"$CI_PROJECT_DIR = {os.environ['CI_PROJECT_DIR']}")
+    folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
+    logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
+    shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
+
+    logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
+    shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
+
+    logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
+    os.remove(f"{CLR_TEST_DIR}/db/export.sql")
+
+    logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
+    os.remove(f"{CLR_TEST_DIR}/db/test-data.sql")
+
+    logging.info(f"Copying files from {DATA_ROOT_DIR}/{folders[index]}/db/ to {CLR_TEST_DIR}/db/")
+    shutil.copytree(f"{DATA_ROOT_DIR}/{folders[index]}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
+
+    logging.info(f"Copying files from {DATA_ROOT_DIR}/{folders[index]}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
+    shutil.copytree(f"{DATA_ROOT_DIR}/{folders[index]}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
+    remove_files_with_extensions(".xml")
+    change_date(folders[index])
+    run_docker(folders[index])
+
+
+def main():
+    folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
+
+    for folder in folders:
+        logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
+        shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
+
+        logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
+        shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
+
+        logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
+        os.remove(f"{CLR_TEST_DIR}/db/export.sql")
+
+        logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
+        os.remove(f"{CLR_TEST_DIR}/db/test-data.sql")
+
+        logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/")
+        shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
+
+        logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
+        shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
+        remove_files_with_extensions(".xml")
+        change_date(folder)
+        run_docker(folder)
+
+
+if __name__=='__main__':
+    main()