sftp integration added for test pipeline

clr_test/clearing-all/clearing_testing_utils/clearing-tester/application.properties

@@ -3,14 +3,14 @@ server.port=9900
 clearing-tester.store.root-dir=/opt/clearing/clearing_testing_utils/store
 
 #sftp storage settings
-clearing-tester.sftp.dbf.importer.in-dir=/opt/clearing/files/dbf/importer/in
-clearing-tester.sftp.dbf.exporter.out-dir=/opt/clearing/files/dbf/exporter
+clearing-tester.sftp.dbf.importer.in-dir=/in/prc/
+clearing-tester.sftp.dbf.exporter.out-dir=/out/prc/
 
-clearing-tester.sftp.swt.importer.in-dir=/opt/clearing/files/swt/importer/in
-clearing-tester.sftp.swt.exporter.out-dir=/opt/clearing/files/swt/exporter
+clearing-tester.sftp.swt.importer.in-dir=/in/rdc/
+clearing-tester.sftp.swt.exporter.out-dir=/out/rdc/
 
-clearing-tester.sftp.xml.importer.in-dir=/opt/clearing/files/xml/importer/in
-clearing-tester.sftp.xml.exporter.out-dir=/opt/clearing/files/xml/exporter
+clearing-tester.sftp.xml.importer.in-dir=/in/prc/
+clearing-tester.sftp.xml.exporter.out-dir=/out/prc/
 
 clearing-tester.sftp.user=user
 clearing-tester.sftp.password=Aa111111

clr_test/clearing-all/clearing_testing_utils/clearing-tester/clearing-tester.sh

@@ -3,6 +3,6 @@
 CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
 cd $CLEARING_TEST_HOME/clearing-tester
 
-CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ --spring.profiles.active=dev"
+CMD="java -jar clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/"
 
 $CMD

clr_test/clearing-all/launch_all.sh

@@ -15,10 +15,10 @@ java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEAR
 java -jar $CLEARING_HOME/bin/clearing-service.jar --spring.config.location=$CLEARING_HOME/settings/clearing-service/ &
 java -jar $CLEARING_HOME/bin/scheduler-service.jar --spring.config.location=$CLEARING_HOME/settings/scheduler-service/ &
 
-java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test &
-java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test &
+java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ &
+java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ &
 
-java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ --spring.profiles.active=test &
-java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ --spring.profiles.active=test &
+java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ &
+java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ &
 
 echo all services launched starting. $?

clr_test/clearing-all/settings/dbf-exporter/application.properties

@@ -8,14 +8,15 @@ export-dbf-service.common.encoding=cp866
 export-dbf-service.common.threads-count=10
 
 export-dbf-service.store.local-temp-dir=/opt/clearing/files/dbf/exporter/
-# код валюты должен быть в верхнем регистре, например для рублей - "RUB" (*.RUB=export/rub/).RUB=export/rub/
-export-dbf-service.store.out-pay-val-dir.RUB=/opt/clearing/files/dbf/exporter/
-export-dbf-service.store.out-pay-val-dir.EUR=/opt/clearing/files/dbf/exporter/
-export-dbf-service.store.out-pay-val-dir.CNY=/opt/clearing/files/dbf/exporter/
-#export-dbf-service.store.user:tester
-#export-dbf-service.store.password=password
-#export-dbf-service.store.server-ip=10.230.238.53
-#export-dbf-service.store.server-port=2222
+
+export-dbf-service.store.out-pay-val-dir.RUB=/out/prc/
+export-dbf-service.store.out-pay-val-dir.EUR=/out/prc/
+export-dbf-service.store.out-pay-val-dir.CNY=/out/prc/
+
+export-dbf-service.store.user=user
+export-dbf-service.store.password=Aa111111
+export-dbf-service.store.server-ip=sftp
+export-dbf-service.store.server-port=22
 
 export-dbf-service.kafka-consumer.bootstrap-servers=kafka:9092
 export-dbf-service.kafka-consumer.group-id=dev-group-balance-service

clr_test/clearing-all/settings/dbf-importer/application.properties

@@ -12,9 +12,10 @@ import-dbf-service.common.insert-batch-size=100
 import-dbf-service.common.threads-count=10
 
 #sftpSrcDir
-import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=clearing_importer_sftp/rub/
-import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=clearing_importer_sftp/eur/
-import-dbf-service.store.sftp-in.sftp-src-dir=clearing_importer_sftp
+import-dbf-service.store.sftp-in.sftp-src-dir=/in/prc/
+import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.rub=/in/prc/
+import-dbf-service.store.sftp-in.sftp-src-pay-val-dir.eur=/in/prc/
+
 import-dbf-service.store.sftp-in.user=user
 import-dbf-service.store.sftp-in.password=Aa111111
 import-dbf-service.store.sftp-in.server-ip=sftp
@@ -24,7 +25,6 @@ import-dbf-service.hazelcast.cluster-members=localhost:5701
 import-dbf-service.hazelcast.login=dev
 import-dbf-service.hazelcast.password=dev-pass
 
-
 import-dbf-service.kafka-producer.bootstrap-servers=kafka:9092
 import-dbf-service.kafka-producer.acks=all
 import-dbf-service.kafka-producer.retries=0

clr_test/clearing-all/settings/swt-exporter/application.properties

@@ -24,8 +24,9 @@ export-swt-service.kafka-producer.batch-size=16384
 export-swt-service.kafka-producer.linger-ms=1
 export-swt-service.kafka-producer.buffer-memory=33554432
 
-export-swt-service.sftp-out.sftp-out-dir=swt_out_sftp_directory
+export-swt-service.sftp-out.sftp-out-dir=/out/rdc/
+
 export-swt-service.sftp-out.user=user
-export-swt-service.sftp-out.password=********
-export-swt-service.sftp-out.server-ip=127.0.0.1
+export-swt-service.sftp-out.password=Aa111111
+export-swt-service.sftp-out.server-ip=sftp
 export-swt-service.sftp-out.server-port=22

clr_test/clearing-all/settings/swt-importer/application.properties

@@ -10,10 +10,11 @@ import-swt-service.store.out-dir=/opt/clearing/files/swt/importer/out/
 import-swt-service.store.out-dir-error=/opt/clearing/files/swt/importer/error/
 
 #sftpSrcDir
-import-swt-service.store.sftp-in.sftp-src-dir=/opt/clearing/swt/in
+import-swt-service.store.sftp-in.sftp-src-dir=/in/rdc/
+
 import-swt-service.store.sftp-in.user=user
-import-swt-service.store.sftp-in.password=*********
-import-swt-service.store.sftp-in.server-ip=127.0.0.1
+import-swt-service.store.sftp-in.password=Aa111111
+import-swt-service.store.sftp-in.server-ip=sftp
 import-swt-service.store.sftp-in.server-port=22
 
 import-swt-service.common.encoding-source=cp866
@@ -24,7 +25,6 @@ import-swt-service.hazelcast.cluster-members=localhost:5701
 import-swt-service.hazelcast.login=dev
 import-swt-service.hazelcast.password=dev-pass
 
-
 import-swt-service.kafka-producer.bootstrap-servers=kafka:9092
 import-swt-service.kafka-producer.acks=all
 import-swt-service.kafka-producer.retries=0

clr_test/docker-compose.yml

@@ -56,7 +56,7 @@ services:
   sftp:
     image: mirror.gcr.io/atmoz/sftp
     volumes:
-      - /opt/mfd/clr_test/upload:/home/user/upload
+      - ./sftp_data:/home/user
     command: user:Aa111111:1001
 
 volumes:

clr_test/run_test_docker_containers.py

@@ -14,7 +14,7 @@ CLR_TEST_DIR = os.environ['CLR_TEST_DIR']
 RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
 
 
-def run_docker(folder_name):
+def run_docker(folder_name: str) -> int:
     logging.info(f"Preparing docker-compose for {folder_name}")
     os.environ['CLR_TEST_NAME'] = folder_name
     os.chdir("clr_test")
@@ -42,7 +42,7 @@ def run_docker(folder_name):
     return code
 
 
-def change_date(folder_name):
+def change_date(folder_name: str) -> None:
     pattern = r"((?<=clearing-tester\.mock-date-for-sdf=)|(?<=clearing-tester\.mock-date-for-execution=))(.+)"
 
     with open(f"{DATA_ROOT_DIR}/{folder_name}/date.txt", 'r') as f:
@@ -54,7 +54,7 @@ def change_date(folder_name):
         f.write(file)
 
 
-def remove_files_with_extensions(extension):
+def remove_files_with_extensions(extension: str) -> None:
     for root, dirs, files in os.walk(CLR_TEST_DIR):
         for currentFile in files:
             if currentFile.lower().endswith(extension):
@@ -62,26 +62,35 @@ def remove_files_with_extensions(extension):
                 os.remove(os.path.join(root, currentFile))
 
 
-def remove_if_exists(filepath):
+def remove_if_exists(filepath: str) -> None:
     if os.path.exists(filepath):
         os.remove(filepath)
 
 
-def remove_directory(path):
+def remove_directory(path: str) -> None:
     logging.info(f"Removing directory {path}")
     shutil.rmtree(path)
 
 
-def copy_directory(path_from, path_to):
+def copy_directory(path_from: str, path_to: str) -> None:
     logging.info(f"Copying files from {path_from} to {path_to}")
     shutil.copytree(path_from, path_to, dirs_exist_ok=True)
 
 
-def apply_test_case(folder: str):
+def apply_test_case(folder: str) -> None:
     remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
     remove_directory(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
     remove_directory(f"{CLR_TEST_DIR}/db/data")
 
+    remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/prc")
+    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/prc")
+    remove_directory(f"{CLR_TEST_DIR}/sftp_data/in/rdc")
+    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/in/rdc")
+    remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/prc")
+    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/prc")
+    remove_directory(f"{CLR_TEST_DIR}/sftp_data/out/rdc")
+    os.mkdir(f"{CLR_TEST_DIR}/sftp_data/out/rdc")
+
     copy_directory(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
                    f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
     copy_directory(f"{DATA_ROOT_DIR}/{folder}/data",
@@ -92,14 +101,14 @@ def apply_test_case(folder: str):
     run_docker(folder)
 
 
-def stop_docker_compose():
+def stop_docker_compose() -> None:
     os.chdir("clr_test")
     logging.info("Stopping docker compose")
     subprocess.call('docker compose down', shell=True)
     os.chdir("..")
 
 
-def main():
+def main() -> None:
     folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
 
     for folder in folders: