From 30fc9c2a037ab347c9eb48118dc6c0ffdda49d40 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 20 Mar 2025 10:57:25 +0300 Subject: [PATCH] run_test_docker_containers.py refactoring, artifacts copy algorithm improved --- clr_test/clearing-all/launch_and_test.sh | 107 +++++++++++++++++++++-- clr_test/docker-compose.yml | 4 - clr_test/run_test_docker_containers.py | 41 ++++----- 3 files changed, 115 insertions(+), 37 deletions(-) diff --git a/clr_test/clearing-all/launch_and_test.sh b/clr_test/clearing-all/launch_and_test.sh index fd606c19d..e71dc1fe7 100644 --- a/clr_test/clearing-all/launch_and_test.sh +++ b/clr_test/clearing-all/launch_and_test.sh @@ -16,16 +16,105 @@ sleep 30 echo Run clearing-tester sh ./clearing_testing_utils/clearing-tester/clearing-tester.sh -if [ -n "$CLR_TEST_NAME" ] && [ -d "./bin/log" ]; then - echo "Copying logs to ./bin/log_artifacts/${CLR_TEST_NAME}" - # Copy all contents from ./bin/log to tmp dir - mkdir -p "./bin/tmp_log_artifact" - cp -a ./bin/log/. "./bin/tmp_log_artifact/" - # Move the copied logs into the artifact location - mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" - mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}" +if [ -n "$CLR_TEST_NAME" ] +then + echo "Copying logs to ./bin/log_artifacts/${CLR_TEST_NAME}" + + if [ -d "./bin/log" ] + then + mkdir -p "./bin/tmp_log_artifact/clearing_logs" + cp -a ./bin/log/. "./bin/tmp_log_artifact/clearing_logs/" + rm -r ./bin/log/* || echo "Can't delete content of folder ./bin/log/" + else + echo "./bin/log does not exists, skipping folder log copy" + fi + + if [ -d "./clearing_testing_utils/store/results" ] + then + mkdir -p "./bin/tmp_log_artifact/results" + cp -a ./clearing_testing_utils/store/results/. "./bin/tmp_log_artifact/results/" + rm -r ./clearing_testing_utils/store/results/* || echo "Can't delete content of folder ./clearing_testing_utils/store/results/" + else + echo "./clearing_testing_utils/store/results does not exists, skipping folder log copy" + fi + + if [ -d "./clearing_testing_utils/clearing-tester/logs" ] + then + mkdir -p "./bin/tmp_log_artifact/tester_logs" + cp -a ./clearing_testing_utils/clearing-tester/logs/. "./bin/tmp_log_artifact/tester_logs/" + rm -r ./clearing_testing_utils/clearing-tester/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-tester/logs/" + else + echo "./clearing_testing_utils/clearing-tester/logs does not exists, skipping folder log copy" + fi + + if [ -d "./clearing_testing_utils/clearing-converter/logs" ] + then + mkdir -p "./bin/tmp_log_artifact/converter_logs" + cp -a ./clearing_testing_utils/clearing-converter/logs/. "./bin/tmp_log_artifact/converter_logs/" + rm -r ./clearing_testing_utils/clearing-converter/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-converter/logs/" + else + echo "./clearing_testing_utils/clearing-converter/logs does not exists, skipping folder log copy" + fi + + if [ -d "./clearing_testing_utils/clearing-checker/logs" ] + then + mkdir -p "./bin/tmp_log_artifact/checker_logs" + cp -a ./clearing_testing_utils/clearing-checker/logs/. "./bin/tmp_log_artifact/checker_logs/" + rm -r ./clearing_testing_utils/clearing-checker/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-checker/logs/" + else + echo "./clearing_testing_utils/clearing-checker/logs does not exists, skipping folder log copy" + fi + + if [ -d "./files/swt/importer/out" ] && [ -d "./files/swt/importer/error" ] && [ -d "./files/swt/exporter" ] + then + mkdir -p "./bin/tmp_log_artifact/swt/importer/out" + mkdir -p "./bin/tmp_log_artifact/swt/importer/error" + mkdir -p "./bin/tmp_log_artifact/swt/exporter" + cp -a ./files/swt/importer/out/. "./bin/tmp_log_artifact/swt/importer/out/" + cp -a ./files/swt/importer/error/. "./bin/tmp_log_artifact/swt/importer/error/" + cp -a ./files/swt/exporter/. "./bin/tmp_log_artifact/swt/exporter/" + rm -r ./files/swt/importer/out/* || echo "Can't delete content of folder ./files/swt/importer/out/" + rm -r ./files/swt/importer/error/* || echo "Can't delete content of folder ./files/swt/importer/error/" + rm -r ./files/swt/exporter/* || echo "Can't delete content of folder ./files/swt/exporter/" + else + echo "./files/swt/importer/out or ./files/swt/importer/error or ./files/swt/exporter does not exists, skipping folder log copy" + fi + + if [ -d "./files/dbf/importer/out" ] && [ -d "./files/dbf/importer/error" ] && [ -d "./files/dbf/exporter" ] + then + mkdir -p "./bin/tmp_log_artifact/dbf/importer/out" + mkdir -p "./bin/tmp_log_artifact/dbf/importer/error" + mkdir -p "./bin/tmp_log_artifact/dbf/exporter" + cp -a ./files/dbf/importer/out/. "./bin/tmp_log_artifact/dbf/importer/out/" + cp -a ./files/dbf/importer/error/. "./bin/tmp_log_artifact/dbf/importer/error/" + cp -a ./files/dbf/exporter/. "./bin/tmp_log_artifact/dbf/exporter/" + rm -r ./files/dbf/importer/out/* || echo "Can't delete content of folder ./files/dbf/importer/out/" + rm -r ./files/dbf/importer/error/* || echo "Can't delete content of folder ./files/dbf/importer/error/" + rm -r ./files/dbf/exporter/* || echo "Can't delete content of folder ./files/dbf/exporter/" + else + echo "./files/dbf/importer/out or ./files/dbf/importer/error or ./files/dbf/exporter does not exists, skipping folder log copy" + fi + + if [ -d "./files/xml/importer/out" ] && [ -d "./files/xml/importer/error" ] && [ -d "./files/xml/exporter" ] + then + mkdir -p "./bin/tmp_log_artifact/xml/importer/out" + mkdir -p "./bin/tmp_log_artifact/xml/importer/error" + mkdir -p "./bin/tmp_log_artifact/xml/exporter" + cp -a ./files/xml/importer/out/. "./bin/tmp_log_artifact/xml/importer/out/" + cp -a ./files/xml/importer/error/. "./bin/tmp_log_artifact/xml/importer/error/" + cp -a ./files/xml/exporter/. "./bin/tmp_log_artifact/xml/exporter/" + rm -r ./files/xml/importer/out/* || echo "Can't delete content of folder ./files/xml/importer/out/" + rm -r ./files/xml/importer/error/* || echo "Can't delete content of folder ./files/xml/importer/error/" + rm -r ./files/xml/exporter/* || echo "Can't delete content of folder ./files/xml/exporter/" + else + echo "./files/xml/importer/out or ./files/xml/importer/error or ./files/xml/exporter does not exists, skipping folder log copy" + fi + + # Move the copied logs into the artifact location + mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" + mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}" else - echo "CLR_TEST_NAME is not set or ./bin/log does not exist. Skipping log copy." + echo "CLR_TEST_NAME is not set. Skipping log copy." fi if [ $? -eq 0 ] diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index e9b91e97d..6b90944d8 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -26,10 +26,6 @@ services: kafka: condition: service_started volumes: - - ./mnt/results:/opt/clearing/clearing_testing_utils/store/results - - ./mnt/logs/clearing:/opt/clearing/bin/log - - ./mnt/logs/tester:/opt/clearing/clearing_testing_utils/clearing-tester/logs - - ./mnt/logs/converter:/opt/clearing/clearing_testing_utils/clearing-converter/logs - logs_volume:/opt/clearing/bin/log_artifacts # restart: always diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index bfd0b6510..bae444dc6 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -67,32 +67,25 @@ def remove_if_exists(filepath): os.remove(filepath) +def remove_directory(path): + logging.info(f"Removing directory {path}") + shutil.rmtree(path) + + +def copy_directory(path_from, path_to): + logging.info(f"Copying files from {path_from} to {path_to}") + shutil.copytree(path_from, path_to, dirs_exist_ok=True) + + def apply_test_case(folder: str): - logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected") - shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected") + remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected") + remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") + remove_directory(f"{CLR_TEST_DIR}/db/data") - logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") - shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") - - logging.info( - f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") - shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", - f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True) - - logging.info(f"Removing folder {CLR_TEST_DIR}/db/data") - shutil.rmtree(f"{CLR_TEST_DIR}/db/data") - - logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/data/ to {CLR_TEST_DIR}/db/data/") - shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/data", f"{CLR_TEST_DIR}/db/data", dirs_exist_ok=True) - - # logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql") - # remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql") - # - # logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql") - # remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql") - # - # logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/") - # shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True) + copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", + f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") + copy_directory(f"{DATA_ROOT_DIR}/{folder}/data", + f"{CLR_TEST_DIR}/db/data") remove_files_with_extensions(".xml") # change_date(folder)