db-updater container added for updating db data up to current ddl version

.gitlab-ci.yml

@@ -54,6 +54,7 @@ test:
   script:
     - echo "Installing maven and python" && apk update && apk add maven python3
     - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin
+    - echo "Copying ddl files to clearing-all/db" && cp -r /builds/mfd/clearing/z-distr/target/clearing/db ./clr_test/clearing-all/db
     - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs
     - python3 clr_test/run_test_docker_containers.py
     - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf -

clr_test/db-updater/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine
+
+ADD combined_update_ddl.sql /init.sql
+ADD init.sh /init.sh
+
+RUN apk add --update --no-cache postgresql-client
+RUN chmod +x init.sh
+RUN find . -name "*.sh" | xargs sed -i 's/\r$//'
+
+ENTRYPOINT [ "./init.sh" ]

clr_test/db-updater/init.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+until pg_isready -U clearing -h db -p 5432 -d clearing -t 1; do
+  echo "Waiting for database to be ready..."
+  sleep 2
+done
+
+echo "PSQL version: $(psql --version)"
+echo "DB password is $PGPASSWORD"
+echo "Database connection test:"
+psql -U clearing -h db -p 5432 -d clearing -c "SELECT 'Database is connected!' AS status"
+
+echo "Executing SQL file:"
+cat init.sql
+
+echo "Updating database"
+psql -U clearing -h db -p 5432 -d clearing -f init.sql
+
+tail -f /dev/null

clr_test/docker-compose.yml

@@ -13,6 +13,14 @@ services:
       timeout: 5s
       retries: 100
 
+  db-updater:
+    build: db-updater
+    environment:
+      - PGPASSWORD=Aa111111
+    depends_on:
+      db:
+        condition: service_healthy
+
   clearing_system:
     build:
       context: ./clearing-all
@@ -21,8 +29,8 @@ services:
     environment:
       - CLR_TEST_NAME=${CLR_TEST_NAME}
     depends_on:
-      db:
-        condition: service_healthy
+      db-updater:
+        condition: service_started
       kafka:
         condition: service_started
     volumes:

clr_test/run_test_docker_containers.py

@@ -1,3 +1,4 @@
+from glob import glob
 import logging
 import os
 import re
@@ -12,6 +13,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(me
 DATA_ROOT_DIR = os.environ['CLEARING_TEST_DIR']
 CLR_TEST_DIR = os.environ['CLR_TEST_DIR']
 RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
+DDL_VERSION_FROM = (3, 18, 0)
 
 
 def run_docker(folder_name: str) -> int:
@@ -37,6 +39,9 @@ def run_docker(folder_name: str) -> int:
     logging.info("Removing all clr_test-db images")
     subprocess.call('docker rmi -f clr_test-db', shell=True)
 
+    logging.info("Removing all clr_test-db-updater images")
+    subprocess.call('docker rmi -f clr_test-db-updater', shell=True)
+
     logging.info("Removing all clr_test-clearing_system images")
     subprocess.call('docker rmi -f clr_test-clearing_system', shell=True)
 
@@ -108,9 +113,45 @@ def stop_docker_compose() -> None:
     os.chdir("..")
 
 
+def combine_ddl_updates_into_file():
+    logging.info(f"Combining DDLs...")
+    logging.info(f"Found files = {os.listdir(f'{CLR_TEST_DIR}/clearing-all/db')}")
+    pattern = re.compile(r'updateDDL_(\d+(?:\.\d+)+)\.sql$', re.IGNORECASE)
+
+    files = []
+    for filepath in glob(f"{CLR_TEST_DIR}/clearing-all/db/updateDDL_*.sql"):
+        filename = os.path.basename(filepath)
+        match = pattern.match(filename)
+        if match:
+            version_str = match.group(1)
+            parts = version_str.split('.')
+
+            try:
+                version_numbers = [int(p) for p in parts]
+                padded_version = version_numbers[:3] + [0] * max(0, 3 - len(version_numbers))
+
+                if tuple(padded_version) > DDL_VERSION_FROM:
+                    files.append((version_numbers, padded_version, filepath))
+
+            except ValueError:
+                continue
+
+    files.sort(key=lambda x: x[0])
+    logging.info(f"Found files: {files}")
+
+    with open(f"{CLR_TEST_DIR}/db-updater/combined_update_ddl.sql", 'w', encoding='utf-8') as outfile:
+        for orig_ver, padded_ver, filepath in files:
+            with open(filepath, 'r', encoding='utf-8') as infile:
+                content = infile.read().rstrip('\n')
+                outfile.write(content + '\n\n')
+
+    logging.info(f"Result: {os.listdir(f'{CLR_TEST_DIR}/db-updater')}")
+
+
 def main() -> None:
     folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
 
+    combine_ddl_updates_into_file()
     for folder in folders:
         stop_docker_compose()
         apply_test_case(folder)