sftp integration added for test pipeline

.gitlab-ci.yml

@@ -54,7 +54,7 @@ test:
   script:
     - echo "Installing maven and python" && apk update && apk add maven python3
     - echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin
-#    - echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./clr_test/db/
+    - echo "Removing old logs folder artifacts/logs" && rm -rf artifacts/logs
     - python3 clr_test/run_test_docker_containers.py
     - echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf -
     - echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume

clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties

@@ -3,14 +3,14 @@ server.port=9900
 clearing-tester.store.root-dir=/opt/clearing/clearing_testing_utils/store
 
 #sftp storage settings
-clearing-tester.sftp.dbf.importer.in-dir=/opt/clearing/files/dbf/importer/in
-clearing-tester.sftp.dbf.exporter.out-dir=/opt/clearing/files/dbf/exporter
+clearing-tester.sftp.dbf.importer.in-dir=/in/prc/
+clearing-tester.sftp.dbf.exporter.out-dir=/out/prc/
 
-clearing-tester.sftp.swt.importer.in-dir=/opt/clearing/files/swt/importer/in
-clearing-tester.sftp.swt.exporter.out-dir=/opt/clearing/files/swt/exporter
+clearing-tester.sftp.swt.importer.in-dir=/in/rdc/
+clearing-tester.sftp.swt.exporter.out-dir=/out/rdc/
 
-clearing-tester.sftp.xml.importer.in-dir=/opt/clearing/files/xml/importer/in
-clearing-tester.sftp.xml.exporter.out-dir=/opt/clearing/files/xml/exporter
+clearing-tester.sftp.xml.importer.in-dir=/in/prc/
+clearing-tester.sftp.xml.exporter.out-dir=/out/prc/
 
 clearing-tester.sftp.user=user
 clearing-tester.sftp.password=Aa111111

clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh

@@ -3,6 +3,6 @@
 CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
 cd $CLEARING_TEST_HOME/clearing-tester
 
-CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ --spring.profiles.active=dev"
+CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/"
 
 $CMD

clr_test/clearing-all/launch_all.sh

@@ -15,10 +15,10 @@ java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEAR
 java -jar $CLEARING_HOME/bin/clearing-service.jar --spring.config.location=$CLEARING_HOME/settings/clearing-service/ &
 java -jar $CLEARING_HOME/bin/scheduler-service.jar --spring.config.location=$CLEARING_HOME/settings/scheduler-service/ &
 
-java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test &
-java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test &
+java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ &
+java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ &
 
-java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ --spring.profiles.active=test &
-java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ --spring.profiles.active=test &
+java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ &
+java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ &
 
 echo all services launched starting. $?

clr_test/clearing-all/launch_and_test.sh

@@ -110,9 +110,18 @@ then
     echo "./files/xml/importer/out or ./files/xml/importer/error or ./files/xml/exporter does not exists, skipping folder log copy"
   fi
 
+  if [ -d "./bin/log_artifacts/${CLR_TEST_NAME}" ]
+  then
+    echo "./bin/log_artifacts/${CLR_TEST_NAME} already exists, removing content"
+    rm -rf "./bin/log_artifacts/${CLR_TEST_NAME}/*"
+  else
+    echo "Creating folder ./bin/log_artifacts/${CLR_TEST_NAME}"
+    mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}"
+  fi
+
   # Move the copied logs into the artifact location
-  mkdir -p "./bin/log_artifacts/${CLR_TEST_NAME}"
-  mv ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}"
+  cp -r ./bin/tmp_log_artifact/* "./bin/log_artifacts/${CLR_TEST_NAME}/"
+  rm -rf ./bin/tmp_log_artifact/*
 else
   echo "CLR_TEST_NAME is not set. Skipping log copy."
 fi

clr_test/clearing-all/settings/dbf-exporter/application.properties

@@ -8,14 +8,15 @@ export-dbf-service.common.encoding=cp866
 export-dbf-service.common.threads-count=10
 
 export-dbf-service.store.local-temp-dir=/opt/clearing/files/dbf/exporter/
-# код валюты должен быть в верхнем регистре, например для рублей - "RUB" (*.RUB=export/rub/).RUB=export/rub/
-export-dbf-service.store.out-pay-val-dir.RUB=/opt/clearing/files/dbf/exporter/
-export-dbf-service.store.out-pay-val-dir.EUR=/opt/clearing/files/dbf/exporter/
-export-dbf-service.store.out-pay-val-dir.CNY=/opt/clearing/files/dbf/exporter/
-#export-dbf-service.store.user:tester
-#export-dbf-service.store.password=password
-#export-dbf-service.store.server-ip=10.230.238.53
-#export-dbf-service.store.server-port=2222
+
+export-dbf-service.store.out-pay-val-dir.RUB=/out/prc/
+export-dbf-service.store.out-pay-val-dir.EUR=/out/prc/
+export-dbf-service.store.out-pay-val-dir.CNY=/out/prc/
+
+export-dbf-service.store.user=user
+export-dbf-service.store.password=Aa111111
+export-dbf-service.store.server-ip=sftp
+export-dbf-service.store.server-port=22
 
 export-dbf-service.kafka-consumer.bootstrap-servers=kafka:9092
 export-dbf-service.kafka-consumer.group-id=dev-group-balance-service

clr_test/clearing-all/settings/dbf-importer/application.properties

@@ -12,9 +12,10 @@ import-dbf-service.common.insert-batch-size=100
 import-dbf-service.common.threads-count=10
 
 #sftpSrcDir
-import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=clearing_importer_sftp/rub/
-import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=clearing_importer_sftp/eur/
-import-dbf-service.store.sftp-in.sftp-src-dir=clearing_importer_sftp
+import-dbf-service.store.sftp-in.sftp-src-dir=/in/prc/
+import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=/in/prc/
+import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=/in/prc/
+
 import-dbf-service.store.sftp-in.user=user
 import-dbf-service.store.sftp-in.password=Aa111111
 import-dbf-service.store.sftp-in.server-ip=sftp
@@ -24,7 +25,6 @@ import-dbf-service.hazelcast.cluster-members=localhost:5701
 import-dbf-service.hazelcast.login=dev
 import-dbf-service.hazelcast.password=dev-pass
 
-
 import-dbf-service.kafka-producer.bootstrap-servers=kafka:9092
 import-dbf-service.kafka-producer.acks=all
 import-dbf-service.kafka-producer.retries=0

clr_test/clearing-all/settings/swt-exporter/application.properties

@@ -24,8 +24,9 @@ export-swt-service.kafka-producer.batch-size=16384
 export-swt-service.kafka-producer.linger-ms=1
 export-swt-service.kafka-producer.buffer-memory=33554432
 
-export-swt-service.sftp-out.sftp-out-dir=swt_out_sftp_directory
+export-swt-service.sftp-out.sftp-out-dir=/out/rdc/
+
 export-swt-service.sftp-out.user=user
-export-swt-service.sftp-out.password=********
-export-swt-service.sftp-out.server-ip=127.0.0.1
+export-swt-service.sftp-out.password=Aa111111
+export-swt-service.sftp-out.server-ip=sftp
 export-swt-service.sftp-out.server-port=22

clr_test/clearing-all/settings/swt-importer/application.properties

@@ -10,10 +10,11 @@ import-swt-service.store.out-dir=/opt/clearing/files/swt/importer/out/
 import-swt-service.store.out-dir-error=/opt/clearing/files/swt/importer/error/
 
 #sftpSrcDir
-import-swt-service.store.sftp-in.sftp-src-dir=/opt/clearing/swt/in
+import-swt-service.store.sftp-in.sftp-src-dir=/in/rdc/
+
 import-swt-service.store.sftp-in.user=user
-import-swt-service.store.sftp-in.password=*********
-import-swt-service.store.sftp-in.server-ip=127.0.0.1
+import-swt-service.store.sftp-in.password=Aa111111
+import-swt-service.store.sftp-in.server-ip=sftp
 import-swt-service.store.sftp-in.server-port=22
 
 import-swt-service.common.encoding-source=cp866
@@ -24,7 +25,6 @@ import-swt-service.hazelcast.cluster-members=localhost:5701
 import-swt-service.hazelcast.login=dev
 import-swt-service.hazelcast.password=dev-pass
 
-
 import-swt-service.kafka-producer.bootstrap-servers=kafka:9092
 import-swt-service.kafka-producer.acks=all
 import-swt-service.kafka-producer.retries=0

clr_test/docker-compose.yml

@@ -54,17 +54,8 @@ services:
       - zookeeper
 
   sftp:
-    image: mirror.gcr.io/atmoz/sftp
-    volumes:
-      - /opt/mfd/clr_test/upload:/home/user/upload
-    command: user:Aa111111:1001
+    build: sftp
 
 volumes:
-  db:
-    driver: local
-  zookeeper_data:
-    driver: local
-  kafka_data:
-    driver: local
   logs_volume:
     driver: local

clr_test/run_test_docker_containers.py

@@ -14,16 +14,25 @@ CLR_TEST_DIR = os.environ['CLR_TEST_DIR']
 RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
 
 
-def run_docker(folder_name):
+def run_docker(folder_name: str) -> int:
     logging.info(f"Preparing docker-compose for {folder_name}")
     os.environ['CLR_TEST_NAME'] = folder_name
     os.chdir("clr_test")
 
-    logging.info("Removing all clr_test-db containers")
-    subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
-
-    logging.info("Removing all clr_test-clearing_system containers")
-    subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
+    # logging.info("Removing all clr_test-db containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
+    #
+    # logging.info("Removing all clr_test-clearing_system containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
+    #
+    # logging.info("Removing all sftp containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/atmoz/sftp)', shell=True)
+    #
+    # logging.info("Removing all zookeeper containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64)', shell=True)
+    #
+    # logging.info("Removing all kafka containers")
+    # subprocess.call('docker rm -v -f $(docker ps --all -q --filter ancestor=mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64)', shell=True)
 
     logging.info("Removing all clr_test-db images")
     subprocess.call('docker rmi -f clr_test-db', shell=True)
@@ -33,7 +42,7 @@ def run_docker(folder_name):
 
     logging.info("Starting docker compose")
     code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True)
-    logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results"))
+
     os.chdir("..")
 
     if code != 0:
@@ -42,7 +51,7 @@ def run_docker(folder_name):
     return code
 
 
-def change_date(folder_name):
+def change_date(folder_name: str) -> None:
     pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)"
 
     with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f:
@@ -54,7 +63,7 @@ def change_date(folder_name):
         f.write(file)
 
 
-def remove_files_with_extensions(extension):
+def remove_files_with_extensions(extension: str) -> None:
     for root, dirs, files in os.walk(CLR_TEST_DIR):
         for currentFile in files:
             if currentFile.lower().endswith(extension):
@@ -62,22 +71,22 @@ def remove_files_with_extensions(extension):
                 os.remove(os.path.join(root, currentFile))
 
 
-def remove_if_exists(filepath):
+def remove_if_exists(filepath: str) -> None:
     if os.path.exists(filepath):
         os.remove(filepath)
 
 
-def remove_directory(path):
+def remove_directory(path: str) -> None:
     logging.info(f"Removing directory {path}")
     shutil.rmtree(path)
 
 
-def copy_directory(path_from, path_to):
+def copy_directory(path_from: str, path_to: str) -> None:
     logging.info(f"Copying files from {path_from} to {path_to}")
     shutil.copytree(path_from, path_to, dirs_exist_ok=True)
 
 
-def apply_test_case(folder: str):
+def apply_test_case(folder: str) -> None:
     remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
     remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
     remove_directory(f"{CLR_TEST_DIR}/db/data")
@@ -92,14 +101,14 @@ def apply_test_case(folder: str):
     run_docker(folder)
 
 
-def stop_docker_compose():
+def stop_docker_compose() -> None:
     os.chdir("clr_test")
     logging.info("Stopping docker compose")
     subprocess.call('docker compose down', shell=True)
     os.chdir("..")
 
 
-def main():
+def main() -> None:
     folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
 
     for folder in folders:

clr_test/sftp/Dockerfile

@@ -0,0 +1,11 @@
+FROM mirror.gcr.io/atmoz/sftp
+
+RUN mkdir -p /home/user/in/prc
+RUN mkdir -p /home/user/in/rdc
+RUN mkdir -p /home/user/out/prc
+RUN mkdir -p /home/user/out/rdc
+
+RUN chmod -R 777 /home/user/in
+RUN chmod -R 777 /home/user/out
+
+CMD [ "user:Aa111111:1001" ]