From 16b51c34edaa90a3b3726cea8c668031d2895617 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 20 Mar 2025 17:43:37 +0300 Subject: [PATCH 01/15] sftp integration added for test pipeline --- .../clearing-tester/application.properties | 12 ++++----- .../clearing-tester/clearing-tester.sh | 2 +- clr_test/clearing-all/launch_all.sh | 8 +++--- .../dbf-exporter/application.properties | 17 ++++++------ .../dbf-importer/application.properties | 8 +++--- .../swt-exporter/application.properties | 7 ++--- .../swt-importer/application.properties | 8 +++--- clr_test/docker-compose.yml | 2 +- clr_test/run_test_docker_containers.py | 27 ++++++++++++------- .../clearing => sftp_data/in/prc}/.gitkeep | 0 .../converter => sftp_data/in/rdc}/.gitkeep | 0 .../tester => sftp_data/out/prc}/.gitkeep | 0 .../results => sftp_data/out/rdc}/.gitkeep | 0 13 files changed, 51 insertions(+), 40 deletions(-) rename clr_test/{mnt/logs/clearing => sftp_data/in/prc}/.gitkeep (100%) rename clr_test/{mnt/logs/converter => sftp_data/in/rdc}/.gitkeep (100%) rename clr_test/{mnt/logs/tester => sftp_data/out/prc}/.gitkeep (100%) rename clr_test/{mnt/results => sftp_data/out/rdc}/.gitkeep (100%) diff --git a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties index 76f54f8e4..2fa1ded0d 100644 --- a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties +++ b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties @@ -3,14 +3,14 @@ server.port=9900 clearing-tester.store.root-dir=/opt/clearing/clearing_testing_utils/store #sftp storage settings -clearing-tester.sftp.dbf.importer.in-dir=/opt/clearing/files/dbf/importer/in -clearing-tester.sftp.dbf.exporter.out-dir=/opt/clearing/files/dbf/exporter +clearing-tester.sftp.dbf.importer.in-dir=/in/prc/ +clearing-tester.sftp.dbf.exporter.out-dir=/out/prc/ -clearing-tester.sftp.swt.importer.in-dir=/opt/clearing/files/swt/importer/in -clearing-tester.sftp.swt.exporter.out-dir=/opt/clearing/files/swt/exporter +clearing-tester.sftp.swt.importer.in-dir=/in/rdc/ +clearing-tester.sftp.swt.exporter.out-dir=/out/rdc/ -clearing-tester.sftp.xml.importer.in-dir=/opt/clearing/files/xml/importer/in -clearing-tester.sftp.xml.exporter.out-dir=/opt/clearing/files/xml/exporter +clearing-tester.sftp.xml.importer.in-dir=/in/prc/ +clearing-tester.sftp.xml.exporter.out-dir=/out/prc/ clearing-tester.sftp.user=user clearing-tester.sftp.password=Aa111111 diff --git a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh index 29facc564..ba10f7555 100644 --- a/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh +++ b/clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh @@ -3,6 +3,6 @@ CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/ cd $CLEARING_TEST_HOME/clearing-tester -CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ --spring.profiles.active=dev" +CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/" $CMD diff --git a/clr_test/clearing-all/launch_all.sh b/clr_test/clearing-all/launch_all.sh index 9994c5a2b..b6ec0f048 100644 --- a/clr_test/clearing-all/launch_all.sh +++ b/clr_test/clearing-all/launch_all.sh @@ -15,10 +15,10 @@ java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEAR java -jar $CLEARING_HOME/bin/clearing-service.jar --spring.config.location=$CLEARING_HOME/settings/clearing-service/ & java -jar $CLEARING_HOME/bin/scheduler-service.jar --spring.config.location=$CLEARING_HOME/settings/scheduler-service/ & -java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test & -java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test & +java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ & +java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ & -java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ --spring.profiles.active=test & -java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ --spring.profiles.active=test & +java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ & +java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ & echo all services launched starting. $? diff --git a/clr_test/clearing-all/settings/dbf-exporter/application.properties b/clr_test/clearing-all/settings/dbf-exporter/application.properties index c2d5731b3..30c23a09b 100644 --- a/clr_test/clearing-all/settings/dbf-exporter/application.properties +++ b/clr_test/clearing-all/settings/dbf-exporter/application.properties @@ -8,14 +8,15 @@ export-dbf-service.common.encoding=cp866 export-dbf-service.common.threads-count=10 export-dbf-service.store.local-temp-dir=/opt/clearing/files/dbf/exporter/ -# код валюты должен быть в верхнем регистре, например для рублей - "RUB" (*.RUB=export/rub/).RUB=export/rub/ -export-dbf-service.store.out-pay-val-dir.RUB=/opt/clearing/files/dbf/exporter/ -export-dbf-service.store.out-pay-val-dir.EUR=/opt/clearing/files/dbf/exporter/ -export-dbf-service.store.out-pay-val-dir.CNY=/opt/clearing/files/dbf/exporter/ -#export-dbf-service.store.user:tester -#export-dbf-service.store.password=password -#export-dbf-service.store.server-ip=10.230.238.53 -#export-dbf-service.store.server-port=2222 + +export-dbf-service.store.out-pay-val-dir.RUB=/out/prc/ +export-dbf-service.store.out-pay-val-dir.EUR=/out/prc/ +export-dbf-service.store.out-pay-val-dir.CNY=/out/prc/ + +export-dbf-service.store.user=user +export-dbf-service.store.password=Aa111111 +export-dbf-service.store.server-ip=sftp +export-dbf-service.store.server-port=22 export-dbf-service.kafka-consumer.bootstrap-servers=kafka:9092 export-dbf-service.kafka-consumer.group-id=dev-group-balance-service diff --git a/clr_test/clearing-all/settings/dbf-importer/application.properties b/clr_test/clearing-all/settings/dbf-importer/application.properties index d938ccd89..6c8939221 100644 --- a/clr_test/clearing-all/settings/dbf-importer/application.properties +++ b/clr_test/clearing-all/settings/dbf-importer/application.properties @@ -12,9 +12,10 @@ import-dbf-service.common.insert-batch-size=100 import-dbf-service.common.threads-count=10 #sftpSrcDir -import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=clearing_importer_sftp/rub/ -import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=clearing_importer_sftp/eur/ -import-dbf-service.store.sftp-in.sftp-src-dir=clearing_importer_sftp +import-dbf-service.store.sftp-in.sftp-src-dir=/in/prc/ +import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=/in/prc/ +import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=/in/prc/ + import-dbf-service.store.sftp-in.user=user import-dbf-service.store.sftp-in.password=Aa111111 import-dbf-service.store.sftp-in.server-ip=sftp @@ -24,7 +25,6 @@ import-dbf-service.hazelcast.cluster-members=localhost:5701 import-dbf-service.hazelcast.login=dev import-dbf-service.hazelcast.password=dev-pass - import-dbf-service.kafka-producer.bootstrap-servers=kafka:9092 import-dbf-service.kafka-producer.acks=all import-dbf-service.kafka-producer.retries=0 diff --git a/clr_test/clearing-all/settings/swt-exporter/application.properties b/clr_test/clearing-all/settings/swt-exporter/application.properties index cf69aab1e..fc0c8ea5f 100644 --- a/clr_test/clearing-all/settings/swt-exporter/application.properties +++ b/clr_test/clearing-all/settings/swt-exporter/application.properties @@ -24,8 +24,9 @@ export-swt-service.kafka-producer.batch-size=16384 export-swt-service.kafka-producer.linger-ms=1 export-swt-service.kafka-producer.buffer-memory=33554432 -export-swt-service.sftp-out.sftp-out-dir=swt_out_sftp_directory +export-swt-service.sftp-out.sftp-out-dir=/out/rdc/ + export-swt-service.sftp-out.user=user -export-swt-service.sftp-out.password=******** -export-swt-service.sftp-out.server-ip=127.0.0.1 +export-swt-service.sftp-out.password=Aa111111 +export-swt-service.sftp-out.server-ip=sftp export-swt-service.sftp-out.server-port=22 \ No newline at end of file diff --git a/clr_test/clearing-all/settings/swt-importer/application.properties b/clr_test/clearing-all/settings/swt-importer/application.properties index 34c8cd019..9af529b1f 100644 --- a/clr_test/clearing-all/settings/swt-importer/application.properties +++ b/clr_test/clearing-all/settings/swt-importer/application.properties @@ -10,10 +10,11 @@ import-swt-service.store.out-dir=/opt/clearing/files/swt/importer/out/ import-swt-service.store.out-dir-error=/opt/clearing/files/swt/importer/error/ #sftpSrcDir -import-swt-service.store.sftp-in.sftp-src-dir=/opt/clearing/swt/in +import-swt-service.store.sftp-in.sftp-src-dir=/in/rdc/ + import-swt-service.store.sftp-in.user=user -import-swt-service.store.sftp-in.password=********* -import-swt-service.store.sftp-in.server-ip=127.0.0.1 +import-swt-service.store.sftp-in.password=Aa111111 +import-swt-service.store.sftp-in.server-ip=sftp import-swt-service.store.sftp-in.server-port=22 import-swt-service.common.encoding-source=cp866 @@ -24,7 +25,6 @@ import-swt-service.hazelcast.cluster-members=localhost:5701 import-swt-service.hazelcast.login=dev import-swt-service.hazelcast.password=dev-pass - import-swt-service.kafka-producer.bootstrap-servers=kafka:9092 import-swt-service.kafka-producer.acks=all import-swt-service.kafka-producer.retries=0 diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 6b90944d8..bd24ac4a0 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -56,7 +56,7 @@ services: sftp: image: mirror.gcr.io/atmoz/sftp volumes: - - /opt/mfd/clr_test/upload:/home/user/upload + - ./sftp_data:/home/user command: user:Aa111111:1001 volumes: diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index bae444dc6..c4e9e82ae 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -14,7 +14,7 @@ CLR_TEST_DIR = os.environ['CLR_TEST_DIR'] RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR'] -def run_docker(folder_name): +def run_docker(folder_name: str) -> int: logging.info(f"Preparing docker-compose for {folder_name}") os.environ['CLR_TEST_NAME'] = folder_name os.chdir("clr_test") @@ -42,7 +42,7 @@ def run_docker(folder_name): return code -def change_date(folder_name): +def change_date(folder_name: str) -> None: pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)" with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f: @@ -54,7 +54,7 @@ def change_date(folder_name): f.write(file) -def remove_files_with_extensions(extension): +def remove_files_with_extensions(extension: str) -> None: for root, dirs, files in os.walk(CLR_TEST_DIR): for currentFile in files: if currentFile.lower().endswith(extension): @@ -62,26 +62,35 @@ def remove_files_with_extensions(extension): os.remove(os.path.join(root, currentFile)) -def remove_if_exists(filepath): +def remove_if_exists(filepath: str) -> None: if os.path.exists(filepath): os.remove(filepath) -def remove_directory(path): +def remove_directory(path: str) -> None: logging.info(f"Removing directory {path}") shutil.rmtree(path) -def copy_directory(path_from, path_to): +def copy_directory(path_from: str, path_to: str) -> None: logging.info(f"Copying files from {path_from} to {path_to}") shutil.copytree(path_from, path_to, dirs_exist_ok=True) -def apply_test_case(folder: str): +def apply_test_case(folder: str) -> None: remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected") remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") remove_directory(f"{CLR_TEST_DIR}/db/data") + remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/prc") + os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/prc") + remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/rdc") + os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/rdc") + remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/prc") + os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/prc") + remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc") + os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc") + copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") copy_directory(f"{DATA_ROOT_DIR}/{folder}/data", @@ -92,14 +101,14 @@ def apply_test_case(folder: str): run_docker(folder) -def stop_docker_compose(): +def stop_docker_compose() -> None: os.chdir("clr_test") logging.info("Stopping docker compose") subprocess.call('docker compose down', shell=True) os.chdir("..") -def main(): +def main() -> None: folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] for folder in folders: diff --git a/clr_test/mnt/logs/clearing/.gitkeep b/clr_test/sftp_data/in/prc/.gitkeep similarity index 100% rename from clr_test/mnt/logs/clearing/.gitkeep rename to clr_test/sftp_data/in/prc/.gitkeep diff --git a/clr_test/mnt/logs/converter/.gitkeep b/clr_test/sftp_data/in/rdc/.gitkeep similarity index 100% rename from clr_test/mnt/logs/converter/.gitkeep rename to clr_test/sftp_data/in/rdc/.gitkeep diff --git a/clr_test/mnt/logs/tester/.gitkeep b/clr_test/sftp_data/out/prc/.gitkeep similarity index 100% rename from clr_test/mnt/logs/tester/.gitkeep rename to clr_test/sftp_data/out/prc/.gitkeep diff --git a/clr_test/mnt/results/.gitkeep b/clr_test/sftp_data/out/rdc/.gitkeep similarity index 100% rename from clr_test/mnt/results/.gitkeep rename to clr_test/sftp_data/out/rdc/.gitkeep From a669f34fcc454ea29d8a520377c1f1d26becfa2c Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 20 Mar 2025 18:16:04 +0300 Subject: [PATCH 02/15] fix v1 --- clr_test/run_test_docker_containers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index c4e9e82ae..3676b38ba 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -25,6 +25,9 @@ def run_docker(folder_name: str) -> int: logging.info("Removing all clr_test-clearing_system containers") subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) + logging.info("Removing all sftp containers") + subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) + logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) From 778a04913953091a6b2262b94688ae26de6c7f81 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 20 Mar 2025 18:35:18 +0300 Subject: [PATCH 03/15] fix v2 --- clr_test/docker-compose.yml | 2 ++ clr_test/run_test_docker_containers.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index bd24ac4a0..2e90e0ab6 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -55,6 +55,8 @@ services: sftp: image: mirror.gcr.io/atmoz/sftp + security_opt: + - seccomp:unconfined volumes: - ./sftp_data:/home/user command: user:Aa111111:1001 diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index 3676b38ba..c4e9e82ae 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -25,9 +25,6 @@ def run_docker(folder_name: str) -> int: logging.info("Removing all clr_test-clearing_system containers") subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) - logging.info("Removing all sftp containers") - subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) - logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) From b3f04e678d9007b7688f23400efb3b9eb86b0cff Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 10:24:52 +0300 Subject: [PATCH 04/15] fix v3 --- .gitlab-ci.yml | 2 +- clr_test/clearing-all/files/dbf/in/.gitkeep | 0 clr_test/clearing-all/files/dbf/out/.gitkeep | 0 clr_test/clearing-all/files/swt/in/.gitkeep | 0 clr_test/clearing-all/files/swt/out/.gitkeep | 0 clr_test/clearing-all/files/xml/in/.gitkeep | 0 clr_test/clearing-all/files/xml/out/.gitkeep | 0 clr_test/docker-compose.yml | 8 -------- clr_test/run_test_docker_containers.py | 1 + 9 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 clr_test/clearing-all/files/dbf/in/.gitkeep delete mode 100644 clr_test/clearing-all/files/dbf/out/.gitkeep delete mode 100644 clr_test/clearing-all/files/swt/in/.gitkeep delete mode 100644 clr_test/clearing-all/files/swt/out/.gitkeep delete mode 100644 clr_test/clearing-all/files/xml/in/.gitkeep delete mode 100644 clr_test/clearing-all/files/xml/out/.gitkeep diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8e78cb26..5e187c7f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,7 +54,7 @@ test: script: - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin -# - echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./clr_test/db/ + - chmod -R 777 ./clr_test/sftp_data - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume diff --git a/clr_test/clearing-all/files/dbf/in/.gitkeep b/clr_test/clearing-all/files/dbf/in/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/dbf/out/.gitkeep b/clr_test/clearing-all/files/dbf/out/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/swt/in/.gitkeep b/clr_test/clearing-all/files/swt/in/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/swt/out/.gitkeep b/clr_test/clearing-all/files/swt/out/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/xml/in/.gitkeep b/clr_test/clearing-all/files/xml/in/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/clearing-all/files/xml/out/.gitkeep b/clr_test/clearing-all/files/xml/out/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 2e90e0ab6..094ed5331 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -55,18 +55,10 @@ services: sftp: image: mirror.gcr.io/atmoz/sftp - security_opt: - - seccomp:unconfined volumes: - ./sftp_data:/home/user command: user:Aa111111:1001 volumes: - db: - driver: local - zookeeper_data: - driver: local - kafka_data: - driver: local logs_volume: driver: local \ No newline at end of file diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index c4e9e82ae..5a4b603c6 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -90,6 +90,7 @@ def apply_test_case(folder: str) -> None: os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/prc") remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc") os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc") + subprocess.call(f"chmod -R 777 {CLR_TEST_DIR}/sftp_data", shell=True) copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") From 063877f3ddd04bd25bab9e95eeca62438f159309 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 10:55:21 +0300 Subject: [PATCH 05/15] fix v4 --- .gitlab-ci.yml | 1 - clr_test/docker-compose.yml | 2 +- clr_test/run_test_docker_containers.py | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e187c7f4..29c861515 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,7 +54,6 @@ test: script: - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin - - chmod -R 777 ./clr_test/sftp_data - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 094ed5331..2c843a755 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -57,7 +57,7 @@ services: image: mirror.gcr.io/atmoz/sftp volumes: - ./sftp_data:/home/user - command: user:Aa111111:1001 + command: user:Aa111111:::in,out,in/prc,in/rdc,out/prc,out/rdc volumes: logs_volume: diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index 5a4b603c6..acc96ff1e 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -33,12 +33,14 @@ def run_docker(folder_name: str) -> int: logging.info("Starting docker compose") code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True) - logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results")) + os.chdir("..") if code != 0: sys.exit(code) + logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results")) + return code @@ -90,7 +92,6 @@ def apply_test_case(folder: str) -> None: os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/prc") remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc") os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc") - subprocess.call(f"chmod -R 777 {CLR_TEST_DIR}/sftp_data", shell=True) copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") From 3a9fb12a9828ef08abd9c3622fa07471d3d7f0c1 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 11:17:30 +0300 Subject: [PATCH 06/15] fix v5 --- .gitlab-ci.yml | 1 + clr_test/clearing-all/launch_and_test.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29c861515..bf0bce65b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,7 @@ test: script: - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin + - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume diff --git a/clr_test/clearing-all/launch_and_test.sh b/clr_test/clearing-all/launch_and_test.sh index e71dc1fe7..b1274c269 100644 --- a/clr_test/clearing-all/launch_and_test.sh +++ b/clr_test/clearing-all/launch_and_test.sh @@ -111,7 +111,7 @@ then fi # Move the copied logs into the artifact location - mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" + mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" || rm -rf "./bin/log_artifacts/${CLR_TEST_NAME}/*" mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}" else echo "CLR_TEST_NAME is not set. Skipping log copy." From 7827f4c509dec276f4aa0edd8550e60b46c0f5d6 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 11:20:18 +0300 Subject: [PATCH 07/15] fix v6 --- clr_test/clearing-all/launch_and_test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/clr_test/clearing-all/launch_and_test.sh b/clr_test/clearing-all/launch_and_test.sh index b1274c269..0eeb36e92 100644 --- a/clr_test/clearing-all/launch_and_test.sh +++ b/clr_test/clearing-all/launch_and_test.sh @@ -110,8 +110,14 @@ then echo "./files/xml/importer/out or ./files/xml/importer/error or ./files/xml/exporter does not exists, skipping folder log copy" fi + if [ -d "./bin/log_artifacts/${CLR_TEST_NAME}" ] + then + rm -rf "./bin/log_artifacts/${CLR_TEST_NAME}/*" + else + mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" + fi + # Move the copied logs into the artifact location - mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" || rm -rf "./bin/log_artifacts/${CLR_TEST_NAME}/*" mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}" else echo "CLR_TEST_NAME is not set. Skipping log copy." From e987b4cbdbbbf4fc560a3dc03e7a001631fac92e Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 11:38:29 +0300 Subject: [PATCH 08/15] fix v7 --- clr_test/clearing-all/launch_and_test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clr_test/clearing-all/launch_and_test.sh b/clr_test/clearing-all/launch_and_test.sh index 0eeb36e92..1fe1adc3b 100644 --- a/clr_test/clearing-all/launch_and_test.sh +++ b/clr_test/clearing-all/launch_and_test.sh @@ -112,13 +112,16 @@ then if [ -d "./bin/log_artifacts/${CLR_TEST_NAME}" ] then + echo "./bin/log_artifacts/${CLR_TEST_NAME} already exists, removing content" rm -rf "./bin/log_artifacts/${CLR_TEST_NAME}/*" else + echo "Creating folder ./bin/log_artifacts/${CLR_TEST_NAME}" mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}" fi # Move the copied logs into the artifact location - mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}" + cp -r ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}/" + rm -rf ./bin/tmp_log_artifact/* else echo "CLR_TEST_NAME is not set. Skipping log copy." fi From 5d1d5a2b73f46f453dea8e9508e1ce2b3fa51e3a Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 11:51:04 +0300 Subject: [PATCH 09/15] fix v8 --- clr_test/run_test_docker_containers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index acc96ff1e..1001a0c34 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -39,8 +39,6 @@ def run_docker(folder_name: str) -> int: if code != 0: sys.exit(code) - logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results")) - return code From a4c27b6f88e600bec64f821af813ca087e29bbb6 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 12:53:27 +0300 Subject: [PATCH 10/15] fix v9 --- clr_test/run_test_docker_containers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index 1001a0c34..7c199b36e 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -20,10 +20,13 @@ def run_docker(folder_name: str) -> int: os.chdir("clr_test") logging.info("Removing all clr_test-db containers") - subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) + subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) logging.info("Removing all clr_test-clearing_system containers") - subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) + subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) + + logging.info("Removing all sftp containers") + subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) @@ -91,6 +94,8 @@ def apply_test_case(folder: str) -> None: remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc") os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc") + logging.info(os.listdir(f"{CLR_TEST_DIR}/sftp_data/out/prc")) + copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") copy_directory(f"{DATA_ROOT_DIR}/{folder}/data", From 1f49b41cf325958ad551af470c8c24d17dcc89c0 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 13:26:29 +0300 Subject: [PATCH 11/15] fix v10 --- clr_test/run_test_docker_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index 7c199b36e..d93949c48 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -26,7 +26,7 @@ def run_docker(folder_name: str) -> int: subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) logging.info("Removing all sftp containers") - subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) + subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) From d2aac78f561c9a86417fe752a134630eb8bbdcef Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 13:48:30 +0300 Subject: [PATCH 12/15] fix v11 --- clr_test/run_test_docker_containers.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index d93949c48..930895ceb 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -106,22 +106,17 @@ def apply_test_case(folder: str) -> None: run_docker(folder) -def stop_docker_compose() -> None: +def main() -> None: + folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] + + for folder in folders: + apply_test_case(folder) + os.chdir("clr_test") logging.info("Stopping docker compose") subprocess.call('docker compose down', shell=True) os.chdir("..") -def main() -> None: - folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] - - for folder in folders: - stop_docker_compose() - apply_test_case(folder) - - stop_docker_compose() - - if __name__ == '__main__': main() From 324ba79b9ffe5c19fa3e456e258123dd360de3c1 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 14:02:17 +0300 Subject: [PATCH 13/15] fix v12 --- clr_test/run_test_docker_containers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index 930895ceb..643617bbc 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -28,6 +28,12 @@ def run_docker(folder_name: str) -> int: logging.info("Removing all sftp containers") subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) + logging.info("Removing all zookeeper containers") + subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True) + + logging.info("Removing all kafka containers") + subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True) + logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) From 140b5ab68fdbac8d095574b8d46bcadf9518f472 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 14:20:58 +0300 Subject: [PATCH 14/15] fix v13 --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf0bce65b..81345cab5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,12 +52,14 @@ test: CLR_TEST_DIR: "/builds/mfd/clearing/clr_test" RESULT_FILES_DIR: "/builds/mfd/clearing/clr_test/result-files" script: + - docker compose -f ./clr_test/docker-compose.yml down -v - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume + - docker compose -f ./clr_test/docker-compose.yml down -v artifacts: expire_in: 1 day when: always From f53dadf62b75732fc53a3b7862e7e1d530460e0c Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Fri, 21 Mar 2025 16:44:39 +0300 Subject: [PATCH 15/15] fix v14 --- .gitlab-ci.yml | 2 - clr_test/docker-compose.yml | 5 +-- clr_test/run_test_docker_containers.py | 56 ++++++++++++-------------- clr_test/sftp/Dockerfile | 11 +++++ clr_test/sftp_data/in/prc/.gitkeep | 0 clr_test/sftp_data/in/rdc/.gitkeep | 0 clr_test/sftp_data/out/prc/.gitkeep | 0 clr_test/sftp_data/out/rdc/.gitkeep | 0 8 files changed, 37 insertions(+), 37 deletions(-) create mode 100644 clr_test/sftp/Dockerfile delete mode 100644 clr_test/sftp_data/in/prc/.gitkeep delete mode 100644 clr_test/sftp_data/in/rdc/.gitkeep delete mode 100644 clr_test/sftp_data/out/prc/.gitkeep delete mode 100644 clr_test/sftp_data/out/rdc/.gitkeep diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81345cab5..bf0bce65b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,14 +52,12 @@ test: CLR_TEST_DIR: "/builds/mfd/clearing/clr_test" RESULT_FILES_DIR: "/builds/mfd/clearing/clr_test/result-files" script: - - docker compose -f ./clr_test/docker-compose.yml down -v - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume - - docker compose -f ./clr_test/docker-compose.yml down -v artifacts: expire_in: 1 day when: always diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 2c843a755..2f45520f8 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -54,10 +54,7 @@ services: - zookeeper sftp: - image: mirror.gcr.io/atmoz/sftp - volumes: - - ./sftp_data:/home/user - command: user:Aa111111:::in,out,in/prc,in/rdc,out/prc,out/rdc + build: sftp volumes: logs_volume: diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index 643617bbc..c3e7e8d0f 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -19,20 +19,20 @@ def run_docker(folder_name: str) -> int: os.environ['CLR_TEST_NAME'] = folder_name os.chdir("clr_test") - logging.info("Removing all clr_test-db containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) - - logging.info("Removing all clr_test-clearing_system containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) - - logging.info("Removing all sftp containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) - - logging.info("Removing all zookeeper containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True) - - logging.info("Removing all kafka containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True) + # logging.info("Removing all clr_test-db containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) + # + # logging.info("Removing all clr_test-clearing_system containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) + # + # logging.info("Removing all sftp containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) + # + # logging.info("Removing all zookeeper containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True) + # + # logging.info("Removing all kafka containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True) logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) @@ -91,17 +91,6 @@ def apply_test_case(folder: str) -> None: remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") remove_directory(f"{CLR_TEST_DIR}/db/data") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/prc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/prc") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/rdc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/rdc") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/prc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/prc") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc") - - logging.info(os.listdir(f"{CLR_TEST_DIR}/sftp_data/out/prc")) - copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") copy_directory(f"{DATA_ROOT_DIR}/{folder}/data", @@ -112,17 +101,22 @@ def apply_test_case(folder: str) -> None: run_docker(folder) -def main() -> None: - folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] - - for folder in folders: - apply_test_case(folder) - +def stop_docker_compose() -> None: os.chdir("clr_test") logging.info("Stopping docker compose") subprocess.call('docker compose down', shell=True) os.chdir("..") +def main() -> None: + folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] + + for folder in folders: + stop_docker_compose() + apply_test_case(folder) + + stop_docker_compose() + + if __name__ == '__main__': main() diff --git a/clr_test/sftp/Dockerfile b/clr_test/sftp/Dockerfile new file mode 100644 index 000000000..4a50ab65f --- /dev/null +++ b/clr_test/sftp/Dockerfile @@ -0,0 +1,11 @@ +FROM mirror.gcr.io/atmoz/sftp + +RUN mkdir -p /home/user/in/prc +RUN mkdir -p /home/user/in/rdc +RUN mkdir -p /home/user/out/prc +RUN mkdir -p /home/user/out/rdc + +RUN chmod -R 777 /home/user/in +RUN chmod -R 777 /home/user/out + +CMD [ "user:Aa111111:1001" ] \ No newline at end of file diff --git a/clr_test/sftp_data/in/prc/.gitkeep b/clr_test/sftp_data/in/prc/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/sftp_data/in/rdc/.gitkeep b/clr_test/sftp_data/in/rdc/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/sftp_data/out/prc/.gitkeep b/clr_test/sftp_data/out/prc/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/sftp_data/out/rdc/.gitkeep b/clr_test/sftp_data/out/rdc/.gitkeep deleted file mode 100644 index e69de29bb..000000000