diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81345cab5..bf0bce65b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,14 +52,12 @@ test: CLR_TEST_DIR: "/builds/mfd/clearing/clr_test" RESULT_FILES_DIR: "/builds/mfd/clearing/clr_test/result-files" script: - - docker compose -f ./clr_test/docker-compose.yml down -v - echo "Installing maven and python" && apk update && apk add maven python3 - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs - python3 clr_test/run_test_docker_containers.py - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf - - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume - - docker compose -f ./clr_test/docker-compose.yml down -v artifacts: expire_in: 1 day when: always diff --git a/clr_test/docker-compose.yml b/clr_test/docker-compose.yml index 2c843a755..2f45520f8 100644 --- a/clr_test/docker-compose.yml +++ b/clr_test/docker-compose.yml @@ -54,10 +54,7 @@ services: - zookeeper sftp: - image: mirror.gcr.io/atmoz/sftp - volumes: - - ./sftp_data:/home/user - command: user:Aa111111:::in,out,in/prc,in/rdc,out/prc,out/rdc + build: sftp volumes: logs_volume: diff --git a/clr_test/run_test_docker_containers.py b/clr_test/run_test_docker_containers.py index 643617bbc..c3e7e8d0f 100644 --- a/clr_test/run_test_docker_containers.py +++ b/clr_test/run_test_docker_containers.py @@ -19,20 +19,20 @@ def run_docker(folder_name: str) -> int: os.environ['CLR_TEST_NAME'] = folder_name os.chdir("clr_test") - logging.info("Removing all clr_test-db containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) - - logging.info("Removing all clr_test-clearing_system containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) - - logging.info("Removing all sftp containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) - - logging.info("Removing all zookeeper containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True) - - logging.info("Removing all kafka containers") - subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True) + # logging.info("Removing all clr_test-db containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True) + # + # logging.info("Removing all clr_test-clearing_system containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True) + # + # logging.info("Removing all sftp containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True) + # + # logging.info("Removing all zookeeper containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True) + # + # logging.info("Removing all kafka containers") + # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True) logging.info("Removing all clr_test-db images") subprocess.call('docker rmi -f clr_test-db', shell=True) @@ -91,17 +91,6 @@ def apply_test_case(folder: str) -> None: remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input") remove_directory(f"{CLR_TEST_DIR}/db/data") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/prc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/prc") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/rdc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/rdc") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/prc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/prc") - remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc") - os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc") - - logging.info(os.listdir(f"{CLR_TEST_DIR}/sftp_data/out/prc")) - copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/") copy_directory(f"{DATA_ROOT_DIR}/{folder}/data", @@ -112,17 +101,22 @@ def apply_test_case(folder: str) -> None: run_docker(folder) -def main() -> None: - folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] - - for folder in folders: - apply_test_case(folder) - +def stop_docker_compose() -> None: os.chdir("clr_test") logging.info("Stopping docker compose") subprocess.call('docker compose down', shell=True) os.chdir("..") +def main() -> None: + folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)] + + for folder in folders: + stop_docker_compose() + apply_test_case(folder) + + stop_docker_compose() + + if __name__ == '__main__': main() diff --git a/clr_test/sftp/Dockerfile b/clr_test/sftp/Dockerfile new file mode 100644 index 000000000..4a50ab65f --- /dev/null +++ b/clr_test/sftp/Dockerfile @@ -0,0 +1,11 @@ +FROM mirror.gcr.io/atmoz/sftp + +RUN mkdir -p /home/user/in/prc +RUN mkdir -p /home/user/in/rdc +RUN mkdir -p /home/user/out/prc +RUN mkdir -p /home/user/out/rdc + +RUN chmod -R 777 /home/user/in +RUN chmod -R 777 /home/user/out + +CMD [ "user:Aa111111:1001" ] \ No newline at end of file diff --git a/clr_test/sftp_data/in/prc/.gitkeep b/clr_test/sftp_data/in/prc/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/sftp_data/in/rdc/.gitkeep b/clr_test/sftp_data/in/rdc/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/sftp_data/out/prc/.gitkeep b/clr_test/sftp_data/out/prc/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/clr_test/sftp_data/out/rdc/.gitkeep b/clr_test/sftp_data/out/rdc/.gitkeep deleted file mode 100644 index e69de29bb..000000000