fix v14

.gitlab-ci.yml

@@ -52,14 +52,12 @@ test:
     CLR_TEST_DIR: "/builds/mfd/clearing/clr_test"
     RESULT_FILES_DIR: "/builds/mfd/clearing/clr_test/result-files"
   script:
-    - docker compose -f ./clr_test/docker-compose.yml down -v
     - echo "Installing maven and python" && apk update && apk add maven python3
     - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin
     - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs
     - python3 clr_test/run_test_docker_containers.py
     - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf -
     - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume
-    - docker compose -f ./clr_test/docker-compose.yml down -v
   artifacts:
     expire_in: 1 day
     when: always

clr_test/docker-compose.yml

@@ -54,10 +54,7 @@ services:
       - zookeeper
 
   sftp:
-    image: mirror.gcr.io/atmoz/sftp
-    volumes:
-      - ./sftp_data:/home/user
-    command: user:Aa111111:::in,out,in/prc,in/rdc,out/prc,out/rdc
+    build: sftp
 
 volumes:
   logs_volume:

clr_test/run_test_docker_containers.py

@@ -19,20 +19,20 @@ def run_docker(folder_name: str) -> int:
     os.environ['CLR_TEST_NAME'] = folder_name
     os.chdir("clr_test")
 
-    logging.info("Removing all clr_test-db containers")
-    subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
-
-    logging.info("Removing all clr_test-clearing_system containers")
-    subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
-
-    logging.info("Removing all sftp containers")
-    subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True)
-
-    logging.info("Removing all zookeeper containers")
-    subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True)
-
-    logging.info("Removing all kafka containers")
-    subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True)
+    # logging.info("Removing all clr_test-db containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
+    #
+    # logging.info("Removing all clr_test-clearing_system containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
+    #
+    # logging.info("Removing all sftp containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True)
+    #
+    # logging.info("Removing all zookeeper containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True)
+    #
+    # logging.info("Removing all kafka containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True)
 
     logging.info("Removing all clr_test-db images")
     subprocess.call('docker rmi -f clr_test-db', shell=True)
@@ -91,17 +91,6 @@ def apply_test_case(folder: str) -> None:
     remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
     remove_directory(f"{CLR_TEST_DIR}/db/data")
 
-    remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/prc")
-    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/prc")
-    remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/rdc")
-    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/rdc")
-    remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/prc")
-    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/prc")
-    remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc")
-    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc")
-
-    logging.info(os.listdir(f"{CLR_TEST_DIR}/sftp_data/out/prc"))
-
     copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
                    f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
     copy_directory(f"{DATA_ROOT_DIR}/{folder}/data",
@@ -112,17 +101,22 @@ def apply_test_case(folder: str) -> None:
     run_docker(folder)
 
 
-def main() -> None:
-    folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
-
-    for folder in folders:
-        apply_test_case(folder)
-
+def stop_docker_compose() -> None:
     os.chdir("clr_test")
     logging.info("Stopping docker compose")
     subprocess.call('docker compose down', shell=True)
     os.chdir("..")
 
 
+def main() -> None:
+    folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
+
+    for folder in folders:
+        stop_docker_compose()
+        apply_test_case(folder)
+
+    stop_docker_compose()
+
+
 if __name__ == '__main__':
     main()

clr_test/sftp/Dockerfile

@@ -0,0 +1,11 @@
+FROM mirror.gcr.io/atmoz/sftp
+
+RUN mkdir -p /home/user/in/prc
+RUN mkdir -p /home/user/in/rdc
+RUN mkdir -p /home/user/out/prc
+RUN mkdir -p /home/user/out/rdc
+
+RUN chmod -R 777 /home/user/in
+RUN chmod -R 777 /home/user/out
+
+CMD [ "user:Aa111111:1001" ]