now tests works on prepared database data, account-service added, clearing-tester.jar and swt-dbf-converter.jar updated

.gitlab-ci.yml

@@ -54,16 +54,10 @@ test:
   script:
     - echo "Installing maven and python" && apk update && apk add maven python3
     - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin
-    - echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./clr_test/db/
+#    - echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./clr_test/db/
     - python3 clr_test/run_test_docker_containers.py
-    - mkdir -p artifacts/logs
+    - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf -
-    - echo "copying logs"
+    - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume
-    - pwd
-    - ls -al
-    - docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf -
-    - ls -la artifacts/logs
-    - echo "Cleaning up log volume"
-    - docker volume rm clr_test_logs_volume
   artifacts:
     expire_in: 1 day
     when: always

clr_test/clearing-all/Dockerfile

@@ -23,6 +23,7 @@ RUN chmod +x $CLEARING_HOME/clearing_testing_utils/clearing-tester/*.sh
 RUN mkdir -p $CLEARING_HOME/files/swt/importer/in
 RUN mkdir -p $CLEARING_HOME/files/swt/importer/out
 RUN mkdir -p $CLEARING_HOME/files/swt/importer/error
+RUN mkdir -p $CLEARING_HOME/files/swt/importer/sftp-in
 RUN mkdir -p $CLEARING_HOME/files/swt/exporter
 RUN mkdir -p $CLEARING_HOME/files/dbf/importer/in
 RUN mkdir -p $CLEARING_HOME/files/dbf/importer/out
@@ -37,10 +38,6 @@ RUN mkdir -p $CLEARING_HOME/files/xml/exporter
 RUN mkdir -p $CLEARING_HOME_BIN/log_artifacts
 
 WORKDIR $CLEARING_HOME
-RUN sed -i 's/\r$//' $CLEARING_HOME/*.sh
+RUN find . -name "*.sh" | xargs sed -i 's/\r$//'
-RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/*.sh
-RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-checker/*.sh
-RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-converter/*.sh
-RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-tester/*.sh
 
 ENTRYPOINT [ "./launch_and_test.sh" ]

clr_test/clearing-all/clearing_testing_utils/clearing-checker/clearing-checker.sh

@@ -3,6 +3,6 @@
 CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
 cd $CLEARING_TEST_HOME/clearing-checker
 
-CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9992  -jar dbf-swt-checker-1.0.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-checker/"
+CMD="java -jar dbf-swt-checker-1.0.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-checker/"
 
 $CMD >/dev/null 2>&1 &

clr_test/clearing-all/clearing_testing_utils/clearing-converter/clearing-converter.sh

@@ -3,7 +3,7 @@
 CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
 cd $CLEARING_TEST_HOME/clearing-converter
 
-CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9991  -jar swt-dbf-converter-0.0.1.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-converter/"
+CMD="java -jar swt-dbf-converter.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-converter/"
 
 $CMD >/dev/null 2>&1 &
 

clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties

@@ -19,15 +19,13 @@ clearing-tester.sftp.server-port=22
 
 #converter service url
 clearing-tester.converter-url=http://localhost:9901/converter/api/convert
-clearing-tester.type-of-conversion=DBF-OK
-# DBF-OK, DBF-!OK, DBF--OK, DBF-EMPTY
 
 #converter service url			
 clearing-tester.checker-url=http://localhost:9902/api/checkFiles
 
 #mock dates settings
-clearing-tester.mock-date-for-sdf=2024-07-26
+clearing-tester.mock-date-for-sdf=true
-clearing-tester.mock-date-for-execution=2024-07-26
+clearing-tester.mock-date-for-execution=true
 
 #hazelcast settings
 clearing-tester.hazelcast.cluster-members=127.0.0.1:5701
@@ -39,11 +37,6 @@ clearing-tester.database.url=jdbc:postgresql://db:5432/clearing
 clearing-tester.database.login=clearing
 clearing-tester.database.password=Aa111111
 
-#common settings
-clearing-tester.common.encoding-source=cp866
-clearing-tester.common.insert-batch-size=100
-clearing-tester.common.threads-count=10
-
 #kafka settings
 clearing-tester.kafka-producer.bootstrap-servers=kafka:9092
 clearing-tester.kafka-producer.acks=all
@@ -59,6 +52,3 @@ clearing-tester.kafka-consumer.session-timeout-ms=30000
 clearing-tester.kafka-consumer.auto-offset-reset=latest
 clearing-tester.kafka-consumer.linger-ms=1
 clearing-tester.kafka-consumer.buffer-memory=33554432
-
-clearing-tester.common.session-type=${session_type:TRDT}
-clearing-tester.common.section=${section:FOND}

clr_test/clearing-all/clearing_testing_utils/run_test_env.sh

@@ -5,4 +5,4 @@ cd /opt/clearing/clearing_testing_utils
 sh ./clearing-checker/clearing-checker.sh
 sh ./clearing-converter/clearing-converter.sh
 
-echo Test finish. xit code: $?
+echo Test initialization finished. xit code: $?

clr_test/clearing-all/launch_all.sh

@@ -7,19 +7,16 @@ cd $CLEARING_HOME/bin
 echo Start starting all in $CLEARING_HOME.
 
 java -jar $CLEARING_HOME/bin/imdg.jar --spring.config.location=$CLEARING_HOME/settings/imdg/ &
+
 java -jar $CLEARING_HOME/bin/backend-api.jar --spring.config.location=$CLEARING_HOME/settings/backend-api/ &
-#java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEARING_HOME/settings/account-service/  &
+java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEARING_HOME/settings/account-service/  &
-#java -jar $CLEARING_HOME/bin/balance-service.jar --spring.config.location=$CLEARING_HOME/settings/balance-service/   &
 java -jar $CLEARING_HOME/bin/clearing-service.jar --spring.config.location=$CLEARING_HOME/settings/clearing-service/ &
-java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test &
-#java -jar $CLEARING_HOME/bin/company-service.jar --spring.config.location=$CLEARING_HOME/settings/company-service/   &
-java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test &
-#java -jar $CLEARING_HOME/bin/gateway-api.jar --spring.config.location=$CLEARING_HOME/settings/gateway-api/   &
-#java -jar $CLEARING_HOME/bin/registry-service.jar --spring.config.location=$CLEARING_HOME/settings/registry-service/  &
-#java -jar $CLEARING_HOME/bin/reports-service.jar --spring.config.location=$CLEARING_HOME/settings/reports-service/  &
 java -jar $CLEARING_HOME/bin/scheduler-service.jar --spring.config.location=$CLEARING_HOME/settings/scheduler-service/ &
-#java -jar $CLEARING_HOME/bin/securities-service.jar --spring.config.location=$CLEARING_HOME/settings/securities-service/  &
+
+java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test &
+java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test &
+
 java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ --spring.profiles.active=test &
 java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ --spring.profiles.active=test &
-#java -jar $CLEARING_TEST_HOME/clearing-tester/clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ &
+
 echo all services launched starting. $?

clr_test/clearing-all/settings/account-service/application.properties

@@ -0,0 +1,22 @@
+spring.main.web-application-type=none
+
+account-service.hazelcast.cluster-members=127.0.0.1:5701
+account-service.hazelcast.login=dev
+account-service.hazelcast.password=dev-pass
+
+account-service.kafka-consumer.bootstrap-servers=kafka:9092
+account-service.kafka-consumer.group-id=dev-group-balance-service
+account-service.kafka-consumer.enable-auto-commit=false
+account-service.kafka-consumer.session-timeout-ms=30000
+account-service.kafka-consumer.auto-offset-reset=latest
+account-service.kafka-consumer.linger-ms=1
+account-service.kafka-consumer.buffer-memory=33554432
+
+account-service.kafka-producer.bootstrap-servers=kafka:9092
+account-service.kafka-producer.acks=all
+account-service.kafka-producer.retries=0
+account-service.kafka-producer.batch-size=16384
+account-service.kafka-producer.linger-ms=1
+account-service.kafka-producer.buffer-memory=33554432
+
+account-service.app.registry-unit=L

clr_test/db/Dockerfile

@@ -1,5 +1,3 @@
 FROM mirror.gcr.io/postgres:14.1-alpine
 
-ADD DDL.sql /docker-entrypoint-initdb.d/
+ADD data /var/lib/postgresql/data
-ADD export.sql /docker-entrypoint-initdb.d/
-ADD test-data.sql /docker-entrypoint-initdb.d/

clr_test/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     build: db
@@ -8,10 +7,6 @@ services:
       - POSTGRES_PASSWORD=Aa111111
       - POSTGRES_DB=clearing
       - POSTGRES_HOST_AUTH_METHOD=trust
-    networks:
-      - clearing-net
-    ports:
-      - "5433:5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U clearing -h db -p 5432 -d clearing"]
       interval: 5s
@@ -30,47 +25,32 @@ services:
         condition: service_healthy
       kafka:
         condition: service_started
-    networks:
-      - clearing-net
-    ports:
-      - "7010:7010"
-      - "5701:5701"
-      - "7090:7090"
-      - "7020:7020"
-      - "8080:8080"
     volumes:
       - ./mnt/results:/opt/clearing/clearing_testing_utils/store/results
-      - logs_volume:/opt/clearing/bin/log_artifacts
+      - ./mnt/logs/clearing:/opt/clearing/bin/log
       - ./mnt/logs/tester:/opt/clearing/clearing_testing_utils/clearing-tester/logs
       - ./mnt/logs/converter:/opt/clearing/clearing_testing_utils/clearing-converter/logs
-#      - ./mnt/dbfout:/opt/clearing/files/dbf/exporter/
+      - logs_volume:/opt/clearing/bin/log_artifacts
     # restart: always
 
   zookeeper:
     image: mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64
     container_name: zookeeper
-    networks:
-      - clearing-net
     environment:
       ZOOKEEPER_SERVER_ID: 1
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
-    ports:
-      - "22181:2181"
 
   kafka:
     image: mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64
+    attach: false
     container_name: kafka
-    networks:
-      - clearing-net
-    ports:
-      - "19092:9092"
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:19092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
@@ -79,12 +59,8 @@ services:
 
   sftp:
     image: mirror.gcr.io/atmoz/sftp
-    networks:
-      - clearing-net
     volumes:
       - /opt/mfd/clr_test/upload:/home/user/upload
-    ports:
-      - "2222:22"
     command: user:Aa111111:1001
 
 volumes:
@@ -95,11 +71,4 @@ volumes:
   kafka_data:
     driver: local
   logs_volume:
-    driver: local
+    driver: local
-
-networks:
-  clearing-net:
-    driver: bridge
-    ipam:
-      config:
-        - subnet: 192.168.220.0/24

clr_test/run_test_docker_containers.py

@@ -1,10 +1,9 @@
+import logging
 import os
 import re
-import sys
-import time
 import shutil
 import subprocess
-import logging
+import sys
 
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
@@ -16,14 +15,24 @@ RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
 
 
 def run_docker(folder_name):
-    logging.info(f"Running docker-compose for {folder_name}")
+    logging.info(f"Preparing docker-compose for {folder_name}")
     os.environ['CLR_TEST_NAME'] = folder_name
     os.chdir("clr_test")
+
+    logging.info("Removing all clr_test-db containers")
     subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
+
+    logging.info("Removing all clr_test-clearing_system containers")
     subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
+
+    logging.info("Removing all clr_test-db images")
     subprocess.call('docker rmi -f clr_test-db', shell=True)
+
+    logging.info("Removing all clr_test-clearing_system images")
     subprocess.call('docker rmi -f clr_test-clearing_system', shell=True)
-    code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system', shell=True)
+
+    logging.info("Starting docker compose")
+    code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True)
     logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results"))
     os.chdir("..")
 
@@ -57,61 +66,50 @@ def remove_if_exists(filepath):
     if os.path.exists(filepath):
         os.remove(filepath)
 
-def run_specific(index):
+
-    logging.info(f"$CI_PROJECT_DIR = {os.environ['CI_PROJECT_DIR']}")
+def apply_test_case(folder: str):
-    folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
     logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
     shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
 
     logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
     shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
 
-    logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
+    logging.info(
-    remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
+        f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
+    shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
+                    f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
 
-    logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
+    logging.info(f"Removing folder {CLR_TEST_DIR}/db/data")
-    remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
+    shutil.rmtree(f"{CLR_TEST_DIR}/db/data")
 
-    logging.info(f"Copying files from {DATA_ROOT_DIR}/{folders[index]}/db/ to {CLR_TEST_DIR}/db/")
+    logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/data/ to {CLR_TEST_DIR}/db/data/")
-    shutil.copytree(f"{DATA_ROOT_DIR}/{folders[index]}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
+    shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/data", f"{CLR_TEST_DIR}/db/data", dirs_exist_ok=True)
 
-    logging.info(f"Copying files from {DATA_ROOT_DIR}/{folders[index]}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
+    # logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
-    shutil.copytree(f"{DATA_ROOT_DIR}/{folders[index]}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
+    # remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
+    #
+    # logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
+    # remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
+    #
+    # logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/")
+    # shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
 
     remove_files_with_extensions(".xml")
-    change_date(folders[index])
+    # change_date(folder)
-    run_docker(folders[index])
+    run_docker(folder)
 
 
 def main():
     folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
 
     for folder in folders:
-        logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
+        apply_test_case(folder)
-        shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
-
-        logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
-        shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
-
-        logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
-        remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
-
-        logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
-        remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
-
-        logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/")
-        shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
-
-        logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
-        shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
-        remove_files_with_extensions(".xml")
-        change_date(folder)
-        run_docker(folder)
 
     os.chdir("clr_test")
+    logging.info("Stopping docker compose")
     subprocess.call('docker compose down', shell=True)
     os.chdir("..")
 
-if __name__=='__main__':
+
+if __name__ == '__main__':
     main()
-    # run_specific(6)