import logging import os import re import shutil import subprocess from glob import glob logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") DATA_ROOT_DIR = os.environ['CLEARING_TEST_DIR'] CLR_TEST_DIR = os.environ['CLR_TEST_DIR'] RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR'] DDL_VERSION_FROM = (3, 23, 0) def run_docker(folder_name: str) -> int: logging.info(f"Preparing docker-compose for {folder_name}") os.environ['CLR_TEST_NAME'] = folder_name os.chdir("clr_test") # logging.info("Removing all clr_test-db containers") # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) # # logging.info("Removing all clr_test-clearing_system containers") # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) # # logging.info("Removing all sftp containers") # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) # # logging.info("Removing all zookeeper containers") # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True) # # logging.info("Removing all kafka containers") # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True) logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) logging.info("Removing all clr_test-db-updater images") subprocess.call('docker rmi -f clr_test-db-updater', shell=True) logging.info("Removing all clr_test-clearing_system images") subprocess.call('docker rmi -f clr_test-clearing_system', shell=True) logging.info("Starting docker compose") code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True) os.chdir("..") return code def change_date(folder_name: str) -> None: pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)" with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f: date = f.read() with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'r') as f: file = f.read() re.sub(pattern, date, file) with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'w') as f: f.write(file) def remove_files_with_extensions(extension: str) -> None: for root, dirs, files in os.walk(CLR_TEST_DIR): for currentFile in files: if currentFile.lower().endswith(extension): logging.info(f"Removing file {os.path.join(root, currentFile)}, based on extension {extension}") os.remove(os.path.join(root, currentFile)) def remove_if_exists(filepath: str) -> None: if os.path.exists(filepath): os.remove(filepath) def remove_directory(path: str) -> None: logging.info(f"Removing directory {path}") shutil.rmtree(path) def copy_directory(path_from: str, path_to: str) -> None: logging.info(f"Copying files from {path_from} to {path_to}") shutil.copytree(path_from, path_to, dirs_exist_ok=True) def append_to_log_file(test_case: str, code: int) -> None: with open("clr_test/results.txt", 'a') as f: if code == 0: f.write(f"Test case {test_case} passed successfully\n") else: f.write(f"Test case {test_case} failed\n") def apply_test_case(folder: str) -> None: remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected") remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") remove_directory(f"{CLR_TEST_DIR}/db/data") copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") copy_directory(f"{DATA_ROOT_DIR}/{folder}/data", f"{CLR_TEST_DIR}/db/data") # remove_files_with_extensions(".xml") # change_date(folder) exit_code = run_docker(folder) append_to_log_file(folder, exit_code) def stop_docker_compose() -> None: os.chdir("clr_test") logging.info("Stopping docker compose") subprocess.call('docker compose down', shell=True) os.chdir("..") def combine_ddl_updates_into_file(): logging.info(f"Combining DDLs...") logging.info(f"Found files = {os.listdir(f'{CLR_TEST_DIR}/clearing-all/db')}") pattern = re.compile(r'updateDDL_(\d+(?:\.\d+)+)\.sql$', re.IGNORECASE) files = [] for filepath in glob(f"{CLR_TEST_DIR}/clearing-all/db/updateDDL_*.sql"): filename = os.path.basename(filepath) match = pattern.match(filename) if match: version_str = match.group(1) parts = version_str.split('.') try: version_numbers = [int(p) for p in parts] padded_version = version_numbers[:3] + [0] * max(0, 3 - len(version_numbers)) if tuple(padded_version) > DDL_VERSION_FROM: files.append((version_numbers, padded_version, filepath)) except ValueError: continue files.sort(key=lambda x: x[0]) logging.info(f"Found files: {files}") with open(f"{CLR_TEST_DIR}/db-updater/combined_update_ddl.sql", 'w', encoding='utf-8') as outfile: for orig_ver, padded_ver, filepath in files: with open(filepath, 'r', encoding='utf-8') as infile: content = infile.read().rstrip('\n') outfile.write(content + '\n\n') logging.info(f"Result: {os.listdir(f'{CLR_TEST_DIR}/db-updater')}") def main() -> None: folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] if os.getenv("TEST_CASES") is not None: logging.info(f"Current TEST_CASES: {os.getenv("TEST_CASES")}") selected_test_cases = os.getenv("TEST_CASES").split(";") folders = list(filter(lambda f: f in selected_test_cases, folders)) else: logging.info("TEST_CASES is None, running all test cases") remove_if_exists("clr_test/results.txt") combine_ddl_updates_into_file() for folder in folders: stop_docker_compose() apply_test_case(folder) stop_docker_compose() if __name__ == '__main__': main()