clearing/clr_test/run_test_docker_containers.py

from glob import glob
import logging
import os
import re
import shutil
import subprocess
import sys


logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")


DATA_ROOT_DIR = os.environ['CLEARING_TEST_DIR']
CLR_TEST_DIR = os.environ['CLR_TEST_DIR']
RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
DDL_VERSION_FROM = (3, 17)


def run_docker(folder_name: str) -> int:
    logging.info(f"Preparing docker-compose for {folder_name}")
    os.environ['CLR_TEST_NAME'] = folder_name
    os.chdir("clr_test")

    # logging.info("Removing all clr_test-db containers")
    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
    #
    # logging.info("Removing all clr_test-clearing_system containers")
    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
    #
    # logging.info("Removing all sftp containers")
    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True)
    #
    # logging.info("Removing all zookeeper containers")
    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True)
    #
    # logging.info("Removing all kafka containers")
    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True)

    logging.info("Removing all clr_test-db images")
    subprocess.call('docker rmi -f clr_test-db', shell=True)

    logging.info("Removing all clr_test-clearing_system images")
    subprocess.call('docker rmi -f clr_test-clearing_system', shell=True)

    logging.info("Starting docker compose")
    code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True)

    os.chdir("..")

    if code != 0:
        sys.exit(code)

    return code


def change_date(folder_name: str) -> None:
    pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)"

    with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f:
        date = f.read()
    with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'r') as f:
        file = f.read()
    re.sub(pattern, date, file)
    with open(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/clearing-tester/application.properties", 'w') as f:
        f.write(file)


def remove_files_with_extensions(extension: str) -> None:
    for root, dirs, files in os.walk(CLR_TEST_DIR):
        for currentFile in files:
            if currentFile.lower().endswith(extension):
                logging.info(f"Removing file {os.path.join(root, currentFile)}, based on extension {extension}")
                os.remove(os.path.join(root, currentFile))


def remove_if_exists(filepath: str) -> None:
    if os.path.exists(filepath):
        os.remove(filepath)


def remove_directory(path: str) -> None:
    logging.info(f"Removing directory {path}")
    shutil.rmtree(path)


def copy_directory(path_from: str, path_to: str) -> None:
    logging.info(f"Copying files from {path_from} to {path_to}")
    shutil.copytree(path_from, path_to, dirs_exist_ok=True)


def apply_test_case(folder: str) -> None:
    remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
    remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
    remove_directory(f"{CLR_TEST_DIR}/db/data")

    copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
                   f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
    copy_directory(f"{DATA_ROOT_DIR}/{folder}/data",
                   f"{CLR_TEST_DIR}/db/data")

    remove_files_with_extensions(".xml")
    # change_date(folder)
    run_docker(folder)


def stop_docker_compose() -> None:
    os.chdir("clr_test")
    logging.info("Stopping docker compose")
    subprocess.call('docker compose down', shell=True)
    os.chdir("..")


def combine_ddl_updates_into_file():
    logging.info(f"Combining DDLs...")
    pattern = re.compile(r'updateDDL_(\d+)\.(\d+)\.sql$')

    files = []
    for filepath in glob(f"{CLR_TEST_DIR}/db/updateDDL_*.sql"):
        filename = os.path.basename(filepath)
        match = pattern.search(filename)
        if match:
            major, minor = map(int, match.groups())
            if (major, minor) > DDL_VERSION_FROM:
                files.append(((major, minor), filepath))

    files.sort(key=lambda x: x[0])

    with open(f"{CLR_TEST_DIR}/db-updater/combined_update_ddl.sql", 'w', encoding='utf-8') as outfile:
        for version, filepath in files:
            with open(filepath, 'r', encoding='utf-8') as infile:
                content = infile.read().rstrip('\n')
                outfile.write(content + '\n\n')

    logging.info(f"Result: {os.listdir(f'{CLR_TEST_DIR}/db-updater')}")


def main() -> None:
    folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]

    combine_ddl_updates_into_file()
    for folder in folders:
        stop_docker_compose()
        apply_test_case(folder)

    stop_docker_compose()


if __name__ == '__main__':
    main()