diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b9d493aa..c88640489 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,7 @@ test: script: - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin + - echo "Copying ddl files to clearing-all/db" && cp -r /builds/mfd/clearing/z-distr/target/clearing/db ./clr_test/clearing-all/db - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - diff --git a/clr_test/db-updater/Dockerfile b/clr_test/db-updater/Dockerfile new file mode 100644 index 000000000..246a01c32 --- /dev/null +++ b/clr_test/db-updater/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine + +ADD combined_update_ddl.sql /init.sql +ADD init.sh /init.sh + +RUN apk add --update --no-cache postgresql-client +RUN chmod +x init.sh +RUN find . -name "*.sh" | xargs sed -i 's/\r$//' + +ENTRYPOINT [ "./init.sh" ] \ No newline at end of file diff --git a/clr_test/db-updater/init.sh b/clr_test/db-updater/init.sh new file mode 100644 index 000000000..2f9e91ce4 --- /dev/null +++ b/clr_test/db-updater/init.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +until pg_isready -U clearing -h db -p 5432 -d clearing -t 1; do + echo "Waiting for database to be ready..." + sleep 2 +done + +echo "PSQL version: $(psql --version)" +echo "DB password is $PGPASSWORD" +echo "Database connection test:" +psql -U clearing -h db -p 5432 -d clearing -c "SELECT 'Database is connected!' AS status" + +echo "Executing SQL file:" +cat init.sql + +echo "Updating database" +psql -U clearing -h db -p 5432 -d clearing -f init.sql + +tail -f /dev/null \ No newline at end of file diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 901e85af0..9b27b14af 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -13,6 +13,14 @@ services: timeout: 5s retries: 100 + db-updater: + build: db-updater + environment: + - PGPASSWORD=Aa111111 + depends_on: + db: + condition: service_healthy + clearing_system: build: context: ./clearing-all @@ -21,8 +29,8 @@ services: environment: - CLR_TEST_NAME=${CLR_TEST_NAME} depends_on: - db: - condition: service_healthy + db-updater: + condition: service_started kafka: condition: service_started volumes: diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index c3e7e8d0f..740fae90d 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -1,3 +1,4 @@ +from glob import glob import logging import os import re @@ -12,6 +13,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(me DATA_ROOT_DIR = os.environ['CLEARING_TEST_DIR'] CLR_TEST_DIR = os.environ['CLR_TEST_DIR'] RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR'] +DDL_VERSION_FROM = (3, 18, 0) def run_docker(folder_name: str) -> int: @@ -37,6 +39,9 @@ def run_docker(folder_name: str) -> int: logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) + logging.info("Removing all clr_test-db-updater images") + subprocess.call('docker rmi -f clr_test-db-updater', shell=True) + logging.info("Removing all clr_test-clearing_system images") subprocess.call('docker rmi -f clr_test-clearing_system', shell=True) @@ -108,9 +113,45 @@ def stop_docker_compose() -> None: os.chdir("..") +def combine_ddl_updates_into_file(): + logging.info(f"Combining DDLs...") + logging.info(f"Found files = {os.listdir(f'{CLR_TEST_DIR}/clearing-all/db')}") + pattern = re.compile(r'updateDDL_(\d+(?:\.\d+)+)\.sql$', re.IGNORECASE) + + files = [] + for filepath in glob(f"{CLR_TEST_DIR}/clearing-all/db/updateDDL_*.sql"): + filename = os.path.basename(filepath) + match = pattern.match(filename) + if match: + version_str = match.group(1) + parts = version_str.split('.') + + try: + version_numbers = [int(p) for p in parts] + padded_version = version_numbers[:3] + [0] * max(0, 3 - len(version_numbers)) + + if tuple(padded_version) > DDL_VERSION_FROM: + files.append((version_numbers, padded_version, filepath)) + + except ValueError: + continue + + files.sort(key=lambda x: x[0]) + logging.info(f"Found files: {files}") + + with open(f"{CLR_TEST_DIR}/db-updater/combined_update_ddl.sql", 'w', encoding='utf-8') as outfile: + for orig_ver, padded_ver, filepath in files: + with open(filepath, 'r', encoding='utf-8') as infile: + content = infile.read().rstrip('\n') + outfile.write(content + '\n\n') + + logging.info(f"Result: {os.listdir(f'{CLR_TEST_DIR}/db-updater')}") + + def main() -> None: folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] + combine_ddl_updates_into_file() for folder in folders: stop_docker_compose() apply_test_case(folder)