run_test_docker_containers.py refactoring, artifacts copy algorithm improved

clr_test/clearing-all/launch_and_test.sh

@@ -16,16 +16,105 @@ sleep 30
 echo Run clearing-tester
 sh ./clearing_testing_utils/clearing-tester/clearing-tester.sh
 
-if [ -n "$CLR_TEST_NAME" ] && [ -d "./bin/log" ]; then
+if [ -n "$CLR_TEST_NAME" ]
-    echo "Copying logs to ./bin/log_artifacts/${CLR_TEST_NAME}"
+then
-    # Copy all contents from ./bin/log to tmp dir
+  echo "Copying logs to ./bin/log_artifacts/${CLR_TEST_NAME}"
-    mkdir -p "./bin/tmp_log_artifact"
+
-    cp -a ./bin/log/. "./bin/tmp_log_artifact/"
+  if [ -d "./bin/log" ]
-     # Move the copied logs into the artifact location
+  then
-    mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}"
+    mkdir -p "./bin/tmp_log_artifact/clearing_logs"
-    mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}"
+    cp -a ./bin/log/. "./bin/tmp_log_artifact/clearing_logs/"
+    rm -r ./bin/log/* || echo "Can't delete content of folder ./bin/log/"
+  else
+    echo "./bin/log does not exists, skipping folder log copy"
+  fi
+
+  if [ -d "./clearing_testing_utils/store/results" ]
+  then
+    mkdir -p "./bin/tmp_log_artifact/results"
+    cp -a ./clearing_testing_utils/store/results/. "./bin/tmp_log_artifact/results/"
+    rm -r ./clearing_testing_utils/store/results/* || echo "Can't delete content of folder ./clearing_testing_utils/store/results/"
+  else
+    echo "./clearing_testing_utils/store/results does not exists, skipping folder log copy"
+  fi
+
+  if [ -d "./clearing_testing_utils/clearing-tester/logs" ]
+  then
+    mkdir -p "./bin/tmp_log_artifact/tester_logs"
+    cp -a ./clearing_testing_utils/clearing-tester/logs/. "./bin/tmp_log_artifact/tester_logs/"
+    rm -r ./clearing_testing_utils/clearing-tester/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-tester/logs/"
+  else
+    echo "./clearing_testing_utils/clearing-tester/logs does not exists, skipping folder log copy"
+  fi
+
+  if [ -d "./clearing_testing_utils/clearing-converter/logs" ]
+  then
+    mkdir -p "./bin/tmp_log_artifact/converter_logs"
+    cp -a ./clearing_testing_utils/clearing-converter/logs/. "./bin/tmp_log_artifact/converter_logs/"
+    rm -r ./clearing_testing_utils/clearing-converter/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-converter/logs/"
+  else
+    echo "./clearing_testing_utils/clearing-converter/logs does not exists, skipping folder log copy"
+  fi
+
+  if [ -d "./clearing_testing_utils/clearing-checker/logs" ]
+  then
+    mkdir -p "./bin/tmp_log_artifact/checker_logs"
+    cp -a ./clearing_testing_utils/clearing-checker/logs/. "./bin/tmp_log_artifact/checker_logs/"
+    rm -r ./clearing_testing_utils/clearing-checker/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-checker/logs/"
+  else
+    echo "./clearing_testing_utils/clearing-checker/logs does not exists, skipping folder log copy"
+  fi
+
+  if [ -d "./files/swt/importer/out" ] && [ -d "./files/swt/importer/error" ] && [ -d "./files/swt/exporter" ]
+  then
+    mkdir -p "./bin/tmp_log_artifact/swt/importer/out"
+    mkdir -p "./bin/tmp_log_artifact/swt/importer/error"
+    mkdir -p "./bin/tmp_log_artifact/swt/exporter"
+    cp -a ./files/swt/importer/out/. "./bin/tmp_log_artifact/swt/importer/out/"
+    cp -a ./files/swt/importer/error/. "./bin/tmp_log_artifact/swt/importer/error/"
+    cp -a ./files/swt/exporter/. "./bin/tmp_log_artifact/swt/exporter/"
+    rm -r ./files/swt/importer/out/* || echo "Can't delete content of folder ./files/swt/importer/out/"
+    rm -r ./files/swt/importer/error/* || echo "Can't delete content of folder ./files/swt/importer/error/"
+    rm -r ./files/swt/exporter/* || echo "Can't delete content of folder ./files/swt/exporter/"
+  else
+    echo "./files/swt/importer/out or ./files/swt/importer/error or ./files/swt/exporter does not exists, skipping folder log copy"
+  fi
+
+  if [ -d "./files/dbf/importer/out" ] && [ -d "./files/dbf/importer/error" ] && [ -d "./files/dbf/exporter" ]
+  then
+    mkdir -p "./bin/tmp_log_artifact/dbf/importer/out"
+    mkdir -p "./bin/tmp_log_artifact/dbf/importer/error"
+    mkdir -p "./bin/tmp_log_artifact/dbf/exporter"
+    cp -a ./files/dbf/importer/out/. "./bin/tmp_log_artifact/dbf/importer/out/"
+    cp -a ./files/dbf/importer/error/. "./bin/tmp_log_artifact/dbf/importer/error/"
+    cp -a ./files/dbf/exporter/. "./bin/tmp_log_artifact/dbf/exporter/"
+    rm -r ./files/dbf/importer/out/* || echo "Can't delete content of folder ./files/dbf/importer/out/"
+    rm -r ./files/dbf/importer/error/* || echo "Can't delete content of folder ./files/dbf/importer/error/"
+    rm -r ./files/dbf/exporter/* || echo "Can't delete content of folder ./files/dbf/exporter/"
+  else
+    echo "./files/dbf/importer/out or ./files/dbf/importer/error or ./files/dbf/exporter does not exists, skipping folder log copy"
+  fi
+
+  if [ -d "./files/xml/importer/out" ] && [ -d "./files/xml/importer/error" ] && [ -d "./files/xml/exporter" ]
+  then
+    mkdir -p "./bin/tmp_log_artifact/xml/importer/out"
+    mkdir -p "./bin/tmp_log_artifact/xml/importer/error"
+    mkdir -p "./bin/tmp_log_artifact/xml/exporter"
+    cp -a ./files/xml/importer/out/. "./bin/tmp_log_artifact/xml/importer/out/"
+    cp -a ./files/xml/importer/error/. "./bin/tmp_log_artifact/xml/importer/error/"
+    cp -a ./files/xml/exporter/. "./bin/tmp_log_artifact/xml/exporter/"
+    rm -r ./files/xml/importer/out/* || echo "Can't delete content of folder ./files/xml/importer/out/"
+    rm -r ./files/xml/importer/error/* || echo "Can't delete content of folder ./files/xml/importer/error/"
+    rm -r ./files/xml/exporter/* || echo "Can't delete content of folder ./files/xml/exporter/"
+  else
+    echo "./files/xml/importer/out or ./files/xml/importer/error or ./files/xml/exporter does not exists, skipping folder log copy"
+  fi
+
+  # Move the copied logs into the artifact location
+  mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}"
+  mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}"
 else
-    echo "CLR_TEST_NAME is not set or ./bin/log does not exist. Skipping log copy."
+  echo "CLR_TEST_NAME is not set. Skipping log copy."
 fi
 
 if [ $? -eq 0 ]

clr_test/docker-compose.yml

@@ -26,10 +26,6 @@ services:
       kafka:
         condition: service_started
     volumes:
-      - ./mnt/results:/opt/clearing/clearing_testing_utils/store/results
-      - ./mnt/logs/clearing:/opt/clearing/bin/log
-      - ./mnt/logs/tester:/opt/clearing/clearing_testing_utils/clearing-tester/logs
-      - ./mnt/logs/converter:/opt/clearing/clearing_testing_utils/clearing-converter/logs
       - logs_volume:/opt/clearing/bin/log_artifacts
     # restart: always
 

clr_test/run_test_docker_containers.py

@@ -67,32 +67,25 @@ def remove_if_exists(filepath):
         os.remove(filepath)
 
 
+def remove_directory(path):
+    logging.info(f"Removing directory {path}")
+    shutil.rmtree(path)
+
+
+def copy_directory(path_from, path_to):
+    logging.info(f"Copying files from {path_from} to {path_to}")
+    shutil.copytree(path_from, path_to, dirs_exist_ok=True)
+
+
 def apply_test_case(folder: str):
-    logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
+    remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
-    shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
+    remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
+    remove_directory(f"{CLR_TEST_DIR}/db/data")
 
-    logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
+    copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
-    shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
+                   f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
-
+    copy_directory(f"{DATA_ROOT_DIR}/{folder}/data",
-    logging.info(
+                   f"{CLR_TEST_DIR}/db/data")
-        f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
-    shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
-                    f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
-
-    logging.info(f"Removing folder {CLR_TEST_DIR}/db/data")
-    shutil.rmtree(f"{CLR_TEST_DIR}/db/data")
-
-    logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/data/ to {CLR_TEST_DIR}/db/data/")
-    shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/data", f"{CLR_TEST_DIR}/db/data", dirs_exist_ok=True)
-
-    # logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
-    # remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
-    #
-    # logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
-    # remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
-    #
-    # logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/")
-    # shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
 
     remove_files_with_extensions(".xml")
     # change_date(folder)