175 lines
6.4 KiB
Python
175 lines
6.4 KiB
Python
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
from glob import glob
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
|
|
DATA_ROOT_DIR = os.environ['CLEARING_TEST_DIR']
|
|
CLR_TEST_DIR = os.environ['CLR_TEST_DIR']
|
|
RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
|
|
DDL_VERSION_FROM = (3, 23, 0)
|
|
|
|
|
|
def run_docker(folder_name: str) -> int:
|
|
logging.info(f"Preparing docker-compose for {folder_name}")
|
|
os.environ['CLR_TEST_NAME'] = folder_name
|
|
os.chdir("clr_test")
|
|
|
|
# logging.info("Removing all clr_test-db containers")
|
|
# subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
|
|
#
|
|
# logging.info("Removing all clr_test-clearing_system containers")
|
|
# subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
|
|
#
|
|
# logging.info("Removing all sftp containers")
|
|
# subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True)
|
|
#
|
|
# logging.info("Removing all zookeeper containers")
|
|
# subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True)
|
|
#
|
|
# logging.info("Removing all kafka containers")
|
|
# subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True)
|
|
|
|
logging.info("Removing all clr_test-db images")
|
|
subprocess.call('docker rmi -f clr_test-db', shell=True)
|
|
|
|
logging.info("Removing all clr_test-db-updater images")
|
|
subprocess.call('docker rmi -f clr_test-db-updater', shell=True)
|
|
|
|
logging.info("Removing all clr_test-clearing_system images")
|
|
subprocess.call('docker rmi -f clr_test-clearing_system', shell=True)
|
|
|
|
logging.info("Starting docker compose")
|
|
code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True)
|
|
|
|
os.chdir("..")
|
|
|
|
return code
|
|
|
|
|
|
def change_date(folder_name: str) -> None:
|
|
pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)"
|
|
|
|
with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f:
|
|
date = f.read()
|
|
with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'r') as f:
|
|
file = f.read()
|
|
re.sub(pattern, date, file)
|
|
with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'w') as f:
|
|
f.write(file)
|
|
|
|
|
|
def remove_files_with_extensions(extension: str) -> None:
|
|
for root, dirs, files in os.walk(CLR_TEST_DIR):
|
|
for currentFile in files:
|
|
if currentFile.lower().endswith(extension):
|
|
logging.info(f"Removing file {os.path.join(root, currentFile)}, based on extension {extension}")
|
|
os.remove(os.path.join(root, currentFile))
|
|
|
|
|
|
def remove_if_exists(filepath: str) -> None:
|
|
if os.path.exists(filepath):
|
|
os.remove(filepath)
|
|
|
|
|
|
def remove_directory(path: str) -> None:
|
|
logging.info(f"Removing directory {path}")
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def copy_directory(path_from: str, path_to: str) -> None:
|
|
logging.info(f"Copying files from {path_from} to {path_to}")
|
|
shutil.copytree(path_from, path_to, dirs_exist_ok=True)
|
|
|
|
|
|
def append_to_log_file(test_case: str, code: int) -> None:
|
|
with open("clr_test/results.txt", 'a') as f:
|
|
if code == 0:
|
|
f.write(f"Test case {test_case} passed successfully\n")
|
|
else:
|
|
f.write(f"Test case {test_case} failed\n")
|
|
|
|
|
|
def apply_test_case(folder: str) -> None:
|
|
remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
|
|
remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
|
|
remove_directory(f"{CLR_TEST_DIR}/db/data")
|
|
|
|
copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
|
|
f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
|
|
copy_directory(f"{DATA_ROOT_DIR}/{folder}/data",
|
|
f"{CLR_TEST_DIR}/db/data")
|
|
|
|
# remove_files_with_extensions(".xml")
|
|
# change_date(folder)
|
|
exit_code = run_docker(folder)
|
|
append_to_log_file(folder, exit_code)
|
|
|
|
|
|
def stop_docker_compose() -> None:
|
|
os.chdir("clr_test")
|
|
logging.info("Stopping docker compose")
|
|
subprocess.call('docker compose down', shell=True)
|
|
os.chdir("..")
|
|
|
|
|
|
def combine_ddl_updates_into_file():
|
|
logging.info(f"Combining DDLs...")
|
|
logging.info(f"Found files = {os.listdir(f'{CLR_TEST_DIR}/clearing-all/db')}")
|
|
pattern = re.compile(r'updateDDL_(\d+(?:\.\d+)+)\.sql$', re.IGNORECASE)
|
|
|
|
files = []
|
|
for filepath in glob(f"{CLR_TEST_DIR}/clearing-all/db/updateDDL_*.sql"):
|
|
filename = os.path.basename(filepath)
|
|
match = pattern.match(filename)
|
|
if match:
|
|
version_str = match.group(1)
|
|
parts = version_str.split('.')
|
|
|
|
try:
|
|
version_numbers = [int(p) for p in parts]
|
|
padded_version = version_numbers[:3] + [0] * max(0, 3 - len(version_numbers))
|
|
|
|
if tuple(padded_version) > DDL_VERSION_FROM:
|
|
files.append((version_numbers, padded_version, filepath))
|
|
|
|
except ValueError:
|
|
continue
|
|
|
|
files.sort(key=lambda x: x[0])
|
|
logging.info(f"Found files: {files}")
|
|
|
|
with open(f"{CLR_TEST_DIR}/db-updater/combined_update_ddl.sql", 'w', encoding='utf-8') as outfile:
|
|
for orig_ver, padded_ver, filepath in files:
|
|
with open(filepath, 'r', encoding='utf-8') as infile:
|
|
content = infile.read().rstrip('\n')
|
|
outfile.write(content + '\n\n')
|
|
|
|
logging.info(f"Result: {os.listdir(f'{CLR_TEST_DIR}/db-updater')}")
|
|
|
|
|
|
def main() -> None:
|
|
folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
|
|
|
|
if os.getenv("TEST_CASES") is not None:
|
|
logging.info(f"Current TEST_CASES: {os.getenv("TEST_CASES")}")
|
|
selected_test_cases = os.getenv("TEST_CASES").split(";")
|
|
folders = list(filter(lambda f: f in selected_test_cases, folders))
|
|
else:
|
|
logging.info("TEST_CASES is None, running all test cases")
|
|
|
|
remove_if_exists("clr_test/results.txt")
|
|
|
|
combine_ddl_updates_into_file()
|
|
for folder in folders:
|
|
stop_docker_compose()
|
|
apply_test_case(folder)
|
|
|
|
stop_docker_compose()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|