run_test_docker_containers.py refactoring, artifacts copy algorithm improved
This commit is contained in:
Ivan Nikolaev-Axenov
parent
8f30ccad7e
commit
30fc9c2a03
3 changed files with 115 additions and 37 deletions
|
|
@ -16,16 +16,105 @@ sleep 30
|
|||
echo Run clearing-tester
|
||||
sh ./clearing_testing_utils/clearing-tester/clearing-tester.sh
|
||||
|
||||
if [ -n "$CLR_TEST_NAME" ] && [ -d "./bin/log" ]; then
|
||||
echo "Copying logs to ./bin/log_artifacts/${CLR_TEST_NAME}"
|
||||
# Copy all contents from ./bin/log to tmp dir
|
||||
mkdir -p "./bin/tmp_log_artifact"
|
||||
cp -a ./bin/log/. "./bin/tmp_log_artifact/"
|
||||
# Move the copied logs into the artifact location
|
||||
mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}"
|
||||
mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}"
|
||||
if [ -n "$CLR_TEST_NAME" ]
|
||||
then
|
||||
echo "Copying logs to ./bin/log_artifacts/${CLR_TEST_NAME}"
|
||||
|
||||
if [ -d "./bin/log" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/clearing_logs"
|
||||
cp -a ./bin/log/. "./bin/tmp_log_artifact/clearing_logs/"
|
||||
rm -r ./bin/log/* || echo "Can't delete content of folder ./bin/log/"
|
||||
else
|
||||
echo "./bin/log does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
if [ -d "./clearing_testing_utils/store/results" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/results"
|
||||
cp -a ./clearing_testing_utils/store/results/. "./bin/tmp_log_artifact/results/"
|
||||
rm -r ./clearing_testing_utils/store/results/* || echo "Can't delete content of folder ./clearing_testing_utils/store/results/"
|
||||
else
|
||||
echo "./clearing_testing_utils/store/results does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
if [ -d "./clearing_testing_utils/clearing-tester/logs" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/tester_logs"
|
||||
cp -a ./clearing_testing_utils/clearing-tester/logs/. "./bin/tmp_log_artifact/tester_logs/"
|
||||
rm -r ./clearing_testing_utils/clearing-tester/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-tester/logs/"
|
||||
else
|
||||
echo "./clearing_testing_utils/clearing-tester/logs does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
if [ -d "./clearing_testing_utils/clearing-converter/logs" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/converter_logs"
|
||||
cp -a ./clearing_testing_utils/clearing-converter/logs/. "./bin/tmp_log_artifact/converter_logs/"
|
||||
rm -r ./clearing_testing_utils/clearing-converter/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-converter/logs/"
|
||||
else
|
||||
echo "./clearing_testing_utils/clearing-converter/logs does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
if [ -d "./clearing_testing_utils/clearing-checker/logs" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/checker_logs"
|
||||
cp -a ./clearing_testing_utils/clearing-checker/logs/. "./bin/tmp_log_artifact/checker_logs/"
|
||||
rm -r ./clearing_testing_utils/clearing-checker/logs/* || echo "Can't delete content of folder ./clearing_testing_utils/clearing-checker/logs/"
|
||||
else
|
||||
echo "./clearing_testing_utils/clearing-checker/logs does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
if [ -d "./files/swt/importer/out" ] && [ -d "./files/swt/importer/error" ] && [ -d "./files/swt/exporter" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/swt/importer/out"
|
||||
mkdir -p "./bin/tmp_log_artifact/swt/importer/error"
|
||||
mkdir -p "./bin/tmp_log_artifact/swt/exporter"
|
||||
cp -a ./files/swt/importer/out/. "./bin/tmp_log_artifact/swt/importer/out/"
|
||||
cp -a ./files/swt/importer/error/. "./bin/tmp_log_artifact/swt/importer/error/"
|
||||
cp -a ./files/swt/exporter/. "./bin/tmp_log_artifact/swt/exporter/"
|
||||
rm -r ./files/swt/importer/out/* || echo "Can't delete content of folder ./files/swt/importer/out/"
|
||||
rm -r ./files/swt/importer/error/* || echo "Can't delete content of folder ./files/swt/importer/error/"
|
||||
rm -r ./files/swt/exporter/* || echo "Can't delete content of folder ./files/swt/exporter/"
|
||||
else
|
||||
echo "./files/swt/importer/out or ./files/swt/importer/error or ./files/swt/exporter does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
if [ -d "./files/dbf/importer/out" ] && [ -d "./files/dbf/importer/error" ] && [ -d "./files/dbf/exporter" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/dbf/importer/out"
|
||||
mkdir -p "./bin/tmp_log_artifact/dbf/importer/error"
|
||||
mkdir -p "./bin/tmp_log_artifact/dbf/exporter"
|
||||
cp -a ./files/dbf/importer/out/. "./bin/tmp_log_artifact/dbf/importer/out/"
|
||||
cp -a ./files/dbf/importer/error/. "./bin/tmp_log_artifact/dbf/importer/error/"
|
||||
cp -a ./files/dbf/exporter/. "./bin/tmp_log_artifact/dbf/exporter/"
|
||||
rm -r ./files/dbf/importer/out/* || echo "Can't delete content of folder ./files/dbf/importer/out/"
|
||||
rm -r ./files/dbf/importer/error/* || echo "Can't delete content of folder ./files/dbf/importer/error/"
|
||||
rm -r ./files/dbf/exporter/* || echo "Can't delete content of folder ./files/dbf/exporter/"
|
||||
else
|
||||
echo "./files/dbf/importer/out or ./files/dbf/importer/error or ./files/dbf/exporter does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
if [ -d "./files/xml/importer/out" ] && [ -d "./files/xml/importer/error" ] && [ -d "./files/xml/exporter" ]
|
||||
then
|
||||
mkdir -p "./bin/tmp_log_artifact/xml/importer/out"
|
||||
mkdir -p "./bin/tmp_log_artifact/xml/importer/error"
|
||||
mkdir -p "./bin/tmp_log_artifact/xml/exporter"
|
||||
cp -a ./files/xml/importer/out/. "./bin/tmp_log_artifact/xml/importer/out/"
|
||||
cp -a ./files/xml/importer/error/. "./bin/tmp_log_artifact/xml/importer/error/"
|
||||
cp -a ./files/xml/exporter/. "./bin/tmp_log_artifact/xml/exporter/"
|
||||
rm -r ./files/xml/importer/out/* || echo "Can't delete content of folder ./files/xml/importer/out/"
|
||||
rm -r ./files/xml/importer/error/* || echo "Can't delete content of folder ./files/xml/importer/error/"
|
||||
rm -r ./files/xml/exporter/* || echo "Can't delete content of folder ./files/xml/exporter/"
|
||||
else
|
||||
echo "./files/xml/importer/out or ./files/xml/importer/error or ./files/xml/exporter does not exists, skipping folder log copy"
|
||||
fi
|
||||
|
||||
# Move the copied logs into the artifact location
|
||||
mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}"
|
||||
mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}"
|
||||
else
|
||||
echo "CLR_TEST_NAME is not set or ./bin/log does not exist. Skipping log copy."
|
||||
echo "CLR_TEST_NAME is not set. Skipping log copy."
|
||||
fi
|
||||
|
||||
if [ $? -eq 0 ]
|
||||
|
|
|
|||
|
|
@ -26,10 +26,6 @@ services:
|
|||
kafka:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- ./mnt/results:/opt/clearing/clearing_testing_utils/store/results
|
||||
- ./mnt/logs/clearing:/opt/clearing/bin/log
|
||||
- ./mnt/logs/tester:/opt/clearing/clearing_testing_utils/clearing-tester/logs
|
||||
- ./mnt/logs/converter:/opt/clearing/clearing_testing_utils/clearing-converter/logs
|
||||
- logs_volume:/opt/clearing/bin/log_artifacts
|
||||
# restart: always
|
||||
|
||||
|
|
|
|||
|
|
@ -67,32 +67,25 @@ def remove_if_exists(filepath):
|
|||
os.remove(filepath)
|
||||
|
||||
|
||||
def remove_directory(path):
|
||||
logging.info(f"Removing directory {path}")
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
def copy_directory(path_from, path_to):
|
||||
logging.info(f"Copying files from {path_from} to {path_to}")
|
||||
shutil.copytree(path_from, path_to, dirs_exist_ok=True)
|
||||
|
||||
|
||||
def apply_test_case(folder: str):
|
||||
logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
|
||||
shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
|
||||
remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
|
||||
remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
|
||||
remove_directory(f"{CLR_TEST_DIR}/db/data")
|
||||
|
||||
logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
|
||||
shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
|
||||
|
||||
logging.info(
|
||||
f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
|
||||
shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
|
||||
f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
|
||||
|
||||
logging.info(f"Removing folder {CLR_TEST_DIR}/db/data")
|
||||
shutil.rmtree(f"{CLR_TEST_DIR}/db/data")
|
||||
|
||||
logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/data/ to {CLR_TEST_DIR}/db/data/")
|
||||
shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/data", f"{CLR_TEST_DIR}/db/data", dirs_exist_ok=True)
|
||||
|
||||
# logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
|
||||
# remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
|
||||
#
|
||||
# logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
|
||||
# remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
|
||||
#
|
||||
# logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/")
|
||||
# shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
|
||||
copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
|
||||
f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
|
||||
copy_directory(f"{DATA_ROOT_DIR}/{folder}/data",
|
||||
f"{CLR_TEST_DIR}/db/data")
|
||||
|
||||
remove_files_with_extensions(".xml")
|
||||
# change_date(folder)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue