now tests works on prepared database data, account-service added, clearing-tester.jar and swt-dbf-converter.jar updated

This commit is contained in:
Ivan Nikolaev-Axenov 2025-03-18 13:24:17 +03:00
parent 2ebe71b479
commit c76acbf1a0
14 changed files with 84 additions and 119 deletions

View file

@ -54,16 +54,10 @@ test:
script:
- echo "Installing maven and python" && apk update && apk add maven python3
- echo "Copying jar files to clearing-all/bin" && cp -r /builds/mfd/clearing/z-distr/target/clearing/bin ./clr_test/clearing-all/bin
- echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./clr_test/db/
# - echo "Copying DDL.sql to db directory" && cp /builds/mfd/clearing/z-distr/target/clearing/db/DDL.sql ./clr_test/db/
- python3 clr_test/run_test_docker_containers.py
- mkdir -p artifacts/logs
- echo "copying logs"
- pwd
- ls -al
- docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf -
- ls -la artifacts/logs
- echo "Cleaning up log volume"
- docker volume rm clr_test_logs_volume
- echo "Copying logs" && mkdir -p artifacts/logs && docker run --rm -v clr_test_logs_volume:/logs alpine tar -C /logs -cf - . | tar -C artifacts/logs -xf -
- echo "Cleaning up log volume" && docker volume rm clr_test_logs_volume
artifacts:
expire_in: 1 day
when: always

View file

@ -23,6 +23,7 @@ RUN chmod +x $CLEARING_HOME/clearing_testing_utils/clearing-tester/*.sh
RUN mkdir -p $CLEARING_HOME/files/swt/importer/in
RUN mkdir -p $CLEARING_HOME/files/swt/importer/out
RUN mkdir -p $CLEARING_HOME/files/swt/importer/error
RUN mkdir -p $CLEARING_HOME/files/swt/importer/sftp-in
RUN mkdir -p $CLEARING_HOME/files/swt/exporter
RUN mkdir -p $CLEARING_HOME/files/dbf/importer/in
RUN mkdir -p $CLEARING_HOME/files/dbf/importer/out
@ -37,10 +38,6 @@ RUN mkdir -p $CLEARING_HOME/files/xml/exporter
RUN mkdir -p $CLEARING_HOME_BIN/log_artifacts
WORKDIR $CLEARING_HOME
RUN sed -i 's/\r$//' $CLEARING_HOME/*.sh
RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/*.sh
RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-checker/*.sh
RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-converter/*.sh
RUN sed -i 's/\r$//' $CLEARING_HOME/clearing_testing_utils/clearing-tester/*.sh
RUN find . -name "*.sh" | xargs sed -i 's/\r$//'
ENTRYPOINT [ "./launch_and_test.sh" ]

View file

@ -3,6 +3,6 @@
CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
cd $CLEARING_TEST_HOME/clearing-checker
CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9992 -jar dbf-swt-checker-1.0.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-checker/"
CMD="java -jar dbf-swt-checker-1.0.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-checker/"
$CMD >/dev/null 2>&1 &

View file

@ -3,7 +3,7 @@
CLEARING_TEST_HOME=/opt/clearing/clearing_testing_utils/
cd $CLEARING_TEST_HOME/clearing-converter
CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:9991 -jar swt-dbf-converter-0.0.1.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-converter/"
CMD="java -jar swt-dbf-converter.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-converter/"
$CMD >/dev/null 2>&1 &

View file

@ -19,15 +19,13 @@ clearing-tester.sftp.server-port=22
#converter service url
clearing-tester.converter-url=http://localhost:9901/converter/api/convert
clearing-tester.type-of-conversion=DBF-OK
# DBF-OK, DBF-!OK, DBF--OK, DBF-EMPTY
#converter service url
clearing-tester.checker-url=http://localhost:9902/api/checkFiles
#mock dates settings
clearing-tester.mock-date-for-sdf=2024-07-26
clearing-tester.mock-date-for-execution=2024-07-26
clearing-tester.mock-date-for-sdf=true
clearing-tester.mock-date-for-execution=true
#hazelcast settings
clearing-tester.hazelcast.cluster-members=127.0.0.1:5701
@ -39,11 +37,6 @@ clearing-tester.database.url=jdbc:postgresql://db:5432/clearing
clearing-tester.database.login=clearing
clearing-tester.database.password=Aa111111
#common settings
clearing-tester.common.encoding-source=cp866
clearing-tester.common.insert-batch-size=100
clearing-tester.common.threads-count=10
#kafka settings
clearing-tester.kafka-producer.bootstrap-servers=kafka:9092
clearing-tester.kafka-producer.acks=all
@ -59,6 +52,3 @@ clearing-tester.kafka-consumer.session-timeout-ms=30000
clearing-tester.kafka-consumer.auto-offset-reset=latest
clearing-tester.kafka-consumer.linger-ms=1
clearing-tester.kafka-consumer.buffer-memory=33554432
clearing-tester.common.session-type=${session_type:TRDT}
clearing-tester.common.section=${section:FOND}

View file

@ -5,4 +5,4 @@ cd /opt/clearing/clearing_testing_utils
sh ./clearing-checker/clearing-checker.sh
sh ./clearing-converter/clearing-converter.sh
echo Test finish. xit code: $?
echo Test initialization finished. xit code: $?

View file

@ -7,19 +7,16 @@ cd $CLEARING_HOME/bin
echo Start starting all in $CLEARING_HOME.
java -jar $CLEARING_HOME/bin/imdg.jar --spring.config.location=$CLEARING_HOME/settings/imdg/ &
java -jar $CLEARING_HOME/bin/backend-api.jar --spring.config.location=$CLEARING_HOME/settings/backend-api/ &
#java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEARING_HOME/settings/account-service/ &
#java -jar $CLEARING_HOME/bin/balance-service.jar --spring.config.location=$CLEARING_HOME/settings/balance-service/ &
java -jar $CLEARING_HOME/bin/account-service.jar --spring.config.location=$CLEARING_HOME/settings/account-service/ &
java -jar $CLEARING_HOME/bin/clearing-service.jar --spring.config.location=$CLEARING_HOME/settings/clearing-service/ &
java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test &
#java -jar $CLEARING_HOME/bin/company-service.jar --spring.config.location=$CLEARING_HOME/settings/company-service/ &
java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test &
#java -jar $CLEARING_HOME/bin/gateway-api.jar --spring.config.location=$CLEARING_HOME/settings/gateway-api/ &
#java -jar $CLEARING_HOME/bin/registry-service.jar --spring.config.location=$CLEARING_HOME/settings/registry-service/ &
#java -jar $CLEARING_HOME/bin/reports-service.jar --spring.config.location=$CLEARING_HOME/settings/reports-service/ &
java -jar $CLEARING_HOME/bin/scheduler-service.jar --spring.config.location=$CLEARING_HOME/settings/scheduler-service/ &
#java -jar $CLEARING_HOME/bin/securities-service.jar --spring.config.location=$CLEARING_HOME/settings/securities-service/ &
java -jar $CLEARING_HOME/bin/dbf-exporter.jar --spring.config.location=$CLEARING_HOME/settings/dbf-exporter/ --spring.profiles.active=test &
java -jar $CLEARING_HOME/bin/dbf-importer.jar --spring.config.location=$CLEARING_HOME/settings/dbf-importer/ --spring.profiles.active=test &
java -jar $CLEARING_HOME/bin/swt-exporter.jar --spring.config.location=$CLEARING_HOME/settings/swt-exporter/ --spring.profiles.active=test &
java -jar $CLEARING_HOME/bin/swt-importer.jar --spring.config.location=$CLEARING_HOME/settings/swt-importer/ --spring.profiles.active=test &
#java -jar $CLEARING_TEST_HOME/clearing-tester/clearing-tester.jar --spring.config.location=$CLEARING_TEST_HOME/clearing-tester/ &
echo all services launched starting. $?

View file

@ -0,0 +1,22 @@
spring.main.web-application-type=none
account-service.hazelcast.cluster-members=127.0.0.1:5701
account-service.hazelcast.login=dev
account-service.hazelcast.password=dev-pass
account-service.kafka-consumer.bootstrap-servers=kafka:9092
account-service.kafka-consumer.group-id=dev-group-balance-service
account-service.kafka-consumer.enable-auto-commit=false
account-service.kafka-consumer.session-timeout-ms=30000
account-service.kafka-consumer.auto-offset-reset=latest
account-service.kafka-consumer.linger-ms=1
account-service.kafka-consumer.buffer-memory=33554432
account-service.kafka-producer.bootstrap-servers=kafka:9092
account-service.kafka-producer.acks=all
account-service.kafka-producer.retries=0
account-service.kafka-producer.batch-size=16384
account-service.kafka-producer.linger-ms=1
account-service.kafka-producer.buffer-memory=33554432
account-service.app.registry-unit=L

View file

@ -1,5 +1,3 @@
FROM mirror.gcr.io/postgres:14.1-alpine
ADD DDL.sql /docker-entrypoint-initdb.d/
ADD export.sql /docker-entrypoint-initdb.d/
ADD test-data.sql /docker-entrypoint-initdb.d/
ADD data /var/lib/postgresql/data

View file

@ -1,4 +1,3 @@
version: "3"
services:
db:
build: db
@ -8,10 +7,6 @@ services:
- POSTGRES_PASSWORD=Aa111111
- POSTGRES_DB=clearing
- POSTGRES_HOST_AUTH_METHOD=trust
networks:
- clearing-net
ports:
- "5433:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U clearing -h db -p 5432 -d clearing"]
interval: 5s
@ -30,47 +25,32 @@ services:
condition: service_healthy
kafka:
condition: service_started
networks:
- clearing-net
ports:
- "7010:7010"
- "5701:5701"
- "7090:7090"
- "7020:7020"
- "8080:8080"
volumes:
- ./mnt/results:/opt/clearing/clearing_testing_utils/store/results
- logs_volume:/opt/clearing/bin/log_artifacts
- ./mnt/logs/clearing:/opt/clearing/bin/log
- ./mnt/logs/tester:/opt/clearing/clearing_testing_utils/clearing-tester/logs
- ./mnt/logs/converter:/opt/clearing/clearing_testing_utils/clearing-converter/logs
# - ./mnt/dbfout:/opt/clearing/files/dbf/exporter/
- logs_volume:/opt/clearing/bin/log_artifacts
# restart: always
zookeeper:
image: mirror.gcr.io/confluentinc/cp-zookeeper:6.2.3.amd64
container_name: zookeeper
networks:
- clearing-net
environment:
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
ports:
- "22181:2181"
kafka:
image: mirror.gcr.io/confluentinc/cp-kafka:6.2.3.amd64
attach: false
container_name: kafka
networks:
- clearing-net
ports:
- "19092:9092"
environment:
KAFKA_BROKER_ID: 1
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
@ -79,12 +59,8 @@ services:
sftp:
image: mirror.gcr.io/atmoz/sftp
networks:
- clearing-net
volumes:
- /opt/mfd/clr_test/upload:/home/user/upload
ports:
- "2222:22"
command: user:Aa111111:1001
volumes:
@ -95,11 +71,4 @@ volumes:
kafka_data:
driver: local
logs_volume:
driver: local
networks:
clearing-net:
driver: bridge
ipam:
config:
- subnet: 192.168.220.0/24
driver: local

View file

@ -1,10 +1,9 @@
import logging
import os
import re
import sys
import time
import shutil
import subprocess
import logging
import sys
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
@ -16,14 +15,24 @@ RESULT_FILES_DIR = os.environ['RESULT_FILES_DIR']
def run_docker(folder_name):
logging.info(f"Running docker-compose for {folder_name}")
logging.info(f"Preparing docker-compose for {folder_name}")
os.environ['CLR_TEST_NAME'] = folder_name
os.chdir("clr_test")
logging.info("Removing all clr_test-db containers")
subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-db)', shell=True)
logging.info("Removing all clr_test-clearing_system containers")
subprocess.call('docker rm -f $(docker ps --all -q --filter ancestor=clr_test-clearing_system)', shell=True)
logging.info("Removing all clr_test-db images")
subprocess.call('docker rmi -f clr_test-db', shell=True)
logging.info("Removing all clr_test-clearing_system images")
subprocess.call('docker rmi -f clr_test-clearing_system', shell=True)
code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system', shell=True)
logging.info("Starting docker compose")
code = subprocess.call('docker compose up --abort-on-container-exit --exit-code-from clearing_system --build --force-recreate', shell=True)
logging.info(os.listdir(f"{CLR_TEST_DIR}/mnt/results"))
os.chdir("..")
@ -57,61 +66,50 @@ def remove_if_exists(filepath):
if os.path.exists(filepath):
os.remove(filepath)
def run_specific(index):
logging.info(f"$CI_PROJECT_DIR = {os.environ['CI_PROJECT_DIR']}")
folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
def apply_test_case(folder: str):
logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
logging.info(
f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/",
f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
logging.info(f"Removing folder {CLR_TEST_DIR}/db/data")
shutil.rmtree(f"{CLR_TEST_DIR}/db/data")
logging.info(f"Copying files from {DATA_ROOT_DIR}/{folders[index]}/db/ to {CLR_TEST_DIR}/db/")
shutil.copytree(f"{DATA_ROOT_DIR}/{folders[index]}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/data/ to {CLR_TEST_DIR}/db/data/")
shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/data", f"{CLR_TEST_DIR}/db/data", dirs_exist_ok=True)
logging.info(f"Copying files from {DATA_ROOT_DIR}/{folders[index]}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
shutil.copytree(f"{DATA_ROOT_DIR}/{folders[index]}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
# logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
# remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
#
# logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
# remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
#
# logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/")
# shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
remove_files_with_extensions(".xml")
change_date(folders[index])
run_docker(folders[index])
# change_date(folder)
run_docker(folder)
def main():
folders = [i for i in sorted(os.listdir(DATA_ROOT_DIR)) if re.search("\\d{1,}.\\d{1,}.*", i)]
for folder in folders:
logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/expected")
logging.info(f"Removing folder {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
shutil.rmtree(f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/store/input")
logging.info(f"Removing file {CLR_TEST_DIR}/db/export.sql")
remove_if_exists(f"{CLR_TEST_DIR}/db/export.sql")
logging.info(f"Removing file {CLR_TEST_DIR}/db/test-data.sql")
remove_if_exists(f"{CLR_TEST_DIR}/db/test-data.sql")
logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/db/ to {CLR_TEST_DIR}/db/")
shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/db/", f"{CLR_TEST_DIR}/db/", dirs_exist_ok=True)
logging.info(f"Copying files from {DATA_ROOT_DIR}/{folder}/clearing_testing_utils/ to {CLR_TEST_DIR}/clearing-all/clearing_testing_utils/")
shutil.copytree(f"{DATA_ROOT_DIR}/{folder}/clearing_testing_utils/", f"{CLR_TEST_DIR}/clearing-all/clearing_testing_utils/", dirs_exist_ok=True)
remove_files_with_extensions(".xml")
change_date(folder)
run_docker(folder)
apply_test_case(folder)
os.chdir("clr_test")
logging.info("Stopping docker compose")
subprocess.call('docker compose down', shell=True)
os.chdir("..")
if __name__=='__main__':
if __name__ == '__main__':
main()
# run_specific(6)