diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8e78cb26..bf0bce65b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,7 +54,7 @@ test: script: - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin -# - echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./clr_test/db/ + - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume diff --git a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties index 76f54f8e4..2fa1ded0d 100644 --- a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties +++ b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties @@ -3,14 +3,14 @@ server.port=9900 clearing-tester.store.root-dir=/opt/clearing/clearing_testing_utils/store #sftp storage settings -clearing-tester.sftp.dbf.importer.in-dir=/opt/clearing/files/dbf/importer/in -clearing-tester.sftp.dbf.exporter.out-dir=/opt/clearing/files/dbf/exporter +clearing-tester.sftp.dbf.importer.in-dir=/in/prc/ +clearing-tester.sftp.dbf.exporter.out-dir=/out/prc/ -clearing-tester.sftp.swt.importer.in-dir=/opt/clearing/files/swt/importer/in -clearing-tester.sftp.swt.exporter.out-dir=/opt/clearing/files/swt/exporter +clearing-tester.sftp.swt.importer.in-dir=/in/rdc/ +clearing-tester.sftp.swt.exporter.out-dir=/out/rdc/ -clearing-tester.sftp.xml.importer.in-dir=/opt/clearing/files/xml/importer/in -clearing-tester.sftp.xml.exporter.out-dir=/opt/clearing/files/xml/exporter +clearing-tester.sftp.xml.importer.in-dir=/in/prc/ +clearing-tester.sftp.xml.exporter.out-dir=/out/prc/ clearing-tester.sftp.user=user clearing-tester.sftp.password=Aa111111 diff --git a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh index 29facc564..ba10f7555 100644 --- a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh +++ b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh @@ -3,6 +3,6 @@ CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/ cd $CLEARING_TEST_HOME/clearing-tester -CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ --spring.profiles.active=dev" +CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/" $CMD diff --git a/clr_test/clearing-all/files/dbf/in/.gitkeep b/clr_test/clearing-all/files/dbf/in/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/dbf/out/.gitkeep b/clr_test/clearing-all/files/dbf/out/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/swt/in/.gitkeep b/clr_test/clearing-all/files/swt/in/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/swt/out/.gitkeep b/clr_test/clearing-all/files/swt/out/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/xml/in/.gitkeep b/clr_test/clearing-all/files/xml/in/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/xml/out/.gitkeep b/clr_test/clearing-all/files/xml/out/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/launch_all.sh b/clr_test/clearing-all/launch_all.sh index 9994c5a2b..b6ec0f048 100644 --- a/clr_test/clearing-all/launch_all.sh +++ b/clr_test/clearing-all/launch_all.sh @@ -15,10 +15,10 @@ java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEAR java -jar $CLEARING_HOME/bin/clearing-service.jar --spring.config.location=$CLEARING_HOME/settings/clearing-service/ & java -jar $CLEARING_HOME/bin/scheduler-service.jar --spring.config.location=$CLEARING_HOME/settings/scheduler-service/ & -java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test & -java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test & +java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ & +java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ & -java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ --spring.profiles.active=test & -java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ --spring.profiles.active=test & +java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ & +java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ & echo all services launched starting. $? diff --git a/clr_test/clearing-all/launch_and_test.sh b/clr_test/clearing-all/launch_and_test.sh index e71dc1fe7..1fe1adc3b 100644 --- a/clr_test/clearing-all/launch_and_test.sh +++ b/clr_test/clearing-all/launch_and_test.sh @@ -110,9 +110,18 @@ then echo "./files/xml/importer/out or ./files/xml/importer/error or ./files/xml/exporter does not exists, skipping folder log copy" fi + if [ -d "./bin/log_artifacts/${CLR_TEST_NAME}" ] + then + echo "./bin/log_artifacts/${CLR_TEST_NAME} already exists, removing content" + rm -rf "./bin/log_artifacts/${CLR_TEST_NAME}/*" + else + echo "Creating folder ./bin/log_artifacts/${CLR_TEST_NAME}" + mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" + fi + # Move the copied logs into the artifact location - mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" - mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}" + cp -r ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}/" + rm -rf ./bin/tmp_log_artifact/* else echo "CLR_TEST_NAME is not set. Skipping log copy." fi diff --git a/clr_test/clearing-all/settings/dbf-exporter/application.properties b/clr_test/clearing-all/settings/dbf-exporter/application.properties index c2d5731b3..30c23a09b 100644 --- a/clr_test/clearing-all/settings/dbf-exporter/application.properties +++ b/clr_test/clearing-all/settings/dbf-exporter/application.properties @@ -8,14 +8,15 @@ export-dbf-service.common.encoding=cp866 export-dbf-service.common.threads-count=10 export-dbf-service.store.local-temp-dir=/opt/clearing/files/dbf/exporter/ -# код валюты должен быть в верхнем регистре, например для рублей - "RUB" (*.RUB=export/rub/).RUB=export/rub/ -export-dbf-service.store.out-pay-val-dir.RUB=/opt/clearing/files/dbf/exporter/ -export-dbf-service.store.out-pay-val-dir.EUR=/opt/clearing/files/dbf/exporter/ -export-dbf-service.store.out-pay-val-dir.CNY=/opt/clearing/files/dbf/exporter/ -#export-dbf-service.store.user:tester -#export-dbf-service.store.password=password -#export-dbf-service.store.server-ip=10.230.238.53 -#export-dbf-service.store.server-port=2222 + +export-dbf-service.store.out-pay-val-dir.RUB=/out/prc/ +export-dbf-service.store.out-pay-val-dir.EUR=/out/prc/ +export-dbf-service.store.out-pay-val-dir.CNY=/out/prc/ + +export-dbf-service.store.user=user +export-dbf-service.store.password=Aa111111 +export-dbf-service.store.server-ip=sftp +export-dbf-service.store.server-port=22 export-dbf-service.kafka-consumer.bootstrap-servers=kafka:9092 export-dbf-service.kafka-consumer.group-id=dev-group-balance-service diff --git a/clr_test/clearing-all/settings/dbf-importer/application.properties b/clr_test/clearing-all/settings/dbf-importer/application.properties index d938ccd89..6c8939221 100644 --- a/clr_test/clearing-all/settings/dbf-importer/application.properties +++ b/clr_test/clearing-all/settings/dbf-importer/application.properties @@ -12,9 +12,10 @@ import-dbf-service.common.insert-batch-size=100 import-dbf-service.common.threads-count=10 #sftpSrcDir -import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=clearing_importer_sftp/rub/ -import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=clearing_importer_sftp/eur/ -import-dbf-service.store.sftp-in.sftp-src-dir=clearing_importer_sftp +import-dbf-service.store.sftp-in.sftp-src-dir=/in/prc/ +import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=/in/prc/ +import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=/in/prc/ + import-dbf-service.store.sftp-in.user=user import-dbf-service.store.sftp-in.password=Aa111111 import-dbf-service.store.sftp-in.server-ip=sftp @@ -24,7 +25,6 @@ import-dbf-service.hazelcast.cluster-members=localhost:5701 import-dbf-service.hazelcast.login=dev import-dbf-service.hazelcast.password=dev-pass - import-dbf-service.kafka-producer.bootstrap-servers=kafka:9092 import-dbf-service.kafka-producer.acks=all import-dbf-service.kafka-producer.retries=0 diff --git a/clr_test/clearing-all/settings/swt-exporter/application.properties b/clr_test/clearing-all/settings/swt-exporter/application.properties index cf69aab1e..fc0c8ea5f 100644 --- a/clr_test/clearing-all/settings/swt-exporter/application.properties +++ b/clr_test/clearing-all/settings/swt-exporter/application.properties @@ -24,8 +24,9 @@ export-swt-service.kafka-producer.batch-size=16384 export-swt-service.kafka-producer.linger-ms=1 export-swt-service.kafka-producer.buffer-memory=33554432 -export-swt-service.sftp-out.sftp-out-dir=swt_out_sftp_directory +export-swt-service.sftp-out.sftp-out-dir=/out/rdc/ + export-swt-service.sftp-out.user=user -export-swt-service.sftp-out.password=******** -export-swt-service.sftp-out.server-ip=127.0.0.1 +export-swt-service.sftp-out.password=Aa111111 +export-swt-service.sftp-out.server-ip=sftp export-swt-service.sftp-out.server-port=22 \ No newline at end of file diff --git a/clr_test/clearing-all/settings/swt-importer/application.properties b/clr_test/clearing-all/settings/swt-importer/application.properties index 34c8cd019..9af529b1f 100644 --- a/clr_test/clearing-all/settings/swt-importer/application.properties +++ b/clr_test/clearing-all/settings/swt-importer/application.properties @@ -10,10 +10,11 @@ import-swt-service.store.out-dir=/opt/clearing/files/swt/importer/out/ import-swt-service.store.out-dir-error=/opt/clearing/files/swt/importer/error/ #sftpSrcDir -import-swt-service.store.sftp-in.sftp-src-dir=/opt/clearing/swt/in +import-swt-service.store.sftp-in.sftp-src-dir=/in/rdc/ + import-swt-service.store.sftp-in.user=user -import-swt-service.store.sftp-in.password=********* -import-swt-service.store.sftp-in.server-ip=127.0.0.1 +import-swt-service.store.sftp-in.password=Aa111111 +import-swt-service.store.sftp-in.server-ip=sftp import-swt-service.store.sftp-in.server-port=22 import-swt-service.common.encoding-source=cp866 @@ -24,7 +25,6 @@ import-swt-service.hazelcast.cluster-members=localhost:5701 import-swt-service.hazelcast.login=dev import-swt-service.hazelcast.password=dev-pass - import-swt-service.kafka-producer.bootstrap-servers=kafka:9092 import-swt-service.kafka-producer.acks=all import-swt-service.kafka-producer.retries=0 diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 6b90944d8..2f45520f8 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -54,17 +54,8 @@ services: - zookeeper sftp: - image: mirror.gcr.io/atmoz/sftp - volumes: - - /opt/mfd/clr_test/upload:/home/user/upload - command: user:Aa111111:1001 + build: sftp volumes: - db: - driver: local - zookeeper_data: - driver: local - kafka_data: - driver: local logs_volume: driver: local \ No newline at end of file diff --git a/clr_test/mnt/logs/clearing/.gitkeep b/clr_test/mnt/logs/clearing/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/mnt/logs/converter/.gitkeep b/clr_test/mnt/logs/converter/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/mnt/logs/tester/.gitkeep b/clr_test/mnt/logs/tester/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/mnt/results/.gitkeep b/clr_test/mnt/results/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index bae444dc6..c3e7e8d0f 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -14,16 +14,25 @@ CLR_TEST_DIR = os.environ['CLR_TEST_DIR'] RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR'] -def run_docker(folder_name): +def run_docker(folder_name: str) -> int: logging.info(f"Preparing docker-compose for {folder_name}") os.environ['CLR_TEST_NAME'] = folder_name os.chdir("clr_test") - logging.info("Removing all clr_test-db containers") - subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) - - logging.info("Removing all clr_test-clearing_system containers") - subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) + # logging.info("Removing all clr_test-db containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) + # + # logging.info("Removing all clr_test-clearing_system containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) + # + # logging.info("Removing all sftp containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) + # + # logging.info("Removing all zookeeper containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True) + # + # logging.info("Removing all kafka containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True) logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) @@ -33,7 +42,7 @@ def run_docker(folder_name): logging.info("Starting docker compose") code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True) - logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results")) + os.chdir("..") if code != 0: @@ -42,7 +51,7 @@ def run_docker(folder_name): return code -def change_date(folder_name): +def change_date(folder_name: str) -> None: pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)" with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f: @@ -54,7 +63,7 @@ def change_date(folder_name): f.write(file) -def remove_files_with_extensions(extension): +def remove_files_with_extensions(extension: str) -> None: for root, dirs, files in os.walk(CLR_TEST_DIR): for currentFile in files: if currentFile.lower().endswith(extension): @@ -62,22 +71,22 @@ def remove_files_with_extensions(extension): os.remove(os.path.join(root, currentFile)) -def remove_if_exists(filepath): +def remove_if_exists(filepath: str) -> None: if os.path.exists(filepath): os.remove(filepath) -def remove_directory(path): +def remove_directory(path: str) -> None: logging.info(f"Removing directory {path}") shutil.rmtree(path) -def copy_directory(path_from, path_to): +def copy_directory(path_from: str, path_to: str) -> None: logging.info(f"Copying files from {path_from} to {path_to}") shutil.copytree(path_from, path_to, dirs_exist_ok=True) -def apply_test_case(folder: str): +def apply_test_case(folder: str) -> None: remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected") remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") remove_directory(f"{CLR_TEST_DIR}/db/data") @@ -92,14 +101,14 @@ def apply_test_case(folder: str): run_docker(folder) -def stop_docker_compose(): +def stop_docker_compose() -> None: os.chdir("clr_test") logging.info("Stopping docker compose") subprocess.call('docker compose down', shell=True) os.chdir("..") -def main(): +def main() -> None: folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] for folder in folders: diff --git a/clr_test/sftp/Dockerfile b/clr_test/sftp/Dockerfile new file mode 100644 index 000000000..4a50ab65f --- /dev/null +++ b/clr_test/sftp/Dockerfile @@ -0,0 +1,11 @@ +FROM mirror.gcr.io/atmoz/sftp + +RUN mkdir -p /home/user/in/prc +RUN mkdir -p /home/user/in/rdc +RUN mkdir -p /home/user/out/prc +RUN mkdir -p /home/user/out/rdc + +RUN chmod -R 777 /home/user/in +RUN chmod -R 777 /home/user/out + +CMD [ "user:Aa111111:1001" ] \ No newline at end of file