From 039f1f85104de2ec431b48b9b89185ad4ca78ba8 Mon Sep 17 00:00:00 2001 From: ialbert Date: Fri, 13 Oct 2023 16:35:11 +0300 Subject: [PATCH] imdg-hist example --- .../config/BackEndApiImdgConfig.java | 24 ++++ .../config/element/BackendApiSettings.java | 9 ++ .../src/main/resources/application.properties | 3 + .../config/HistoryPoolMapConfigs.java | 15 ++- .../mapstores/AbstractSliceMapLoader.java | 19 +-- .../SearchMoneyBalanceRegisterMapStore.java | 27 +++++ .../services/HazelcastLifecycleSupport.java | 111 ++++++++++++++++++ .../src/main/resources/application.properties | 6 +- .../iml/hazelcast/adapter/ImdgHazelcast.java | 6 + .../java/ru/spcex/platform/imdg/api/Imdg.java | 4 + z-distr/pom.xml | 2 +- 11 files changed, 209 insertions(+), 17 deletions(-) create mode 100644 clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/services/HazelcastLifecycleSupport.java diff --git a/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java index b43a385db..b35c5af31 100644 --- a/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java +++ b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java @@ -21,6 +21,16 @@ public class BackEndApiImdgConfig { return createThreadPoolTaskExecutor(1, false); } + @Bean(name = "taskExecutorHazelcastClientInitializerHist") + public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializerHist() { + return createThreadPoolTaskExecutor(1, true); + } + + @Bean(name = "taskExecutorIdGeneratorAwaiterHist") + public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiterHist() { + return createThreadPoolTaskExecutor(1, false); + } + @Autowired @Bean public ImdgProvider imdgProvider( @@ -35,6 +45,20 @@ public class BackEndApiImdgConfig { return imdg; } + @Autowired + @Bean + public ImdgProvider imdgProviderHist( + @Qualifier("taskExecutorHazelcastClientInitializerHist") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + @Qualifier("taskExecutorIdGeneratorAwaiterHist") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + BackendApiSettings clientSetting + ) { + ImdgProvider imdg = new HazelcastService(taskExecutorHazelcastClientInitializer, + taskExecutorIdGeneratorAwaiter, + clientSetting.getHazelcastSearch()); +// todo корректное ожидание готовности imdg.waitAvailable(); + return imdg; + } + private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) { ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor(); diff --git a/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/element/BackendApiSettings.java b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/element/BackendApiSettings.java index 79ba99f2d..9b51e8890 100644 --- a/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/element/BackendApiSettings.java +++ b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/element/BackendApiSettings.java @@ -13,6 +13,7 @@ import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams; @ConfigurationProperties("backend-api") public class BackendApiSettings { private HazelcastClientParams hazelcast; + private HazelcastClientParams hazelcastSearch; private KafkaProducerSettings kafkaProducer; private KafkaConsumerSettings kafkaConsumer; private SecuritySettings security; @@ -57,4 +58,12 @@ public class BackendApiSettings { public void setSecurity(SecuritySettings security) { this.security = security; } + + public HazelcastClientParams getHazelcastSearch() { + return hazelcastSearch; + } + + public void setHazelcastSearch(HazelcastClientParams hazelcastSearch) { + this.hazelcastSearch = hazelcastSearch; + } } diff --git a/clearing-parent/backend-api/src/main/resources/application.properties b/clearing-parent/backend-api/src/main/resources/application.properties index 29cf01fe5..24016512d 100644 --- a/clearing-parent/backend-api/src/main/resources/application.properties +++ b/clearing-parent/backend-api/src/main/resources/application.properties @@ -9,6 +9,9 @@ backend-api.example-setting=test backend-api.hazelcast.cluster-members=127.0.0.1:5701 backend-api.hazelcast.login=dev backend-api.hazelcast.password=dev-pass +backend-api.hazelcast-search.cluster-members=127.0.0.1:5702 +backend-api.hazelcast-search.login=dev-hist +backend-api.hazelcast-search.password=dev-pass-hist backend-api.kafka-producer.bootstrap-servers=localhost:9092 backend-api.kafka-producer.acks=all backend-api.kafka-producer.retries=0 diff --git a/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/config/HistoryPoolMapConfigs.java b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/config/HistoryPoolMapConfigs.java index ee8f1a414..eb670d92b 100644 --- a/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/config/HistoryPoolMapConfigs.java +++ b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/config/HistoryPoolMapConfigs.java @@ -6,14 +6,12 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import ru.spcex.clearing.imdg.base.AutoconfiguredMap; import ru.spcex.clearing.imdg.base.DictionaryTMapStore; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; +import java.util.*; @Configuration public class HistoryPoolMapConfigs { @@ -91,4 +89,13 @@ public class HistoryPoolMapConfigs { log.debug("Configured {} mapStore's", out.size()); return out; } + + //fixme remove + @Bean("historyMapNames") + public Set historyMaps() { + Set historyMaps = new HashSet<>(); +// historyMaps.add(IMDGDistributedNames.Map_MoneyBalanceRegister); + return historyMaps; + } + } diff --git a/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/AbstractSliceMapLoader.java b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/AbstractSliceMapLoader.java index 15246e269..d278e1f96 100644 --- a/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/AbstractSliceMapLoader.java +++ b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/AbstractSliceMapLoader.java @@ -16,7 +16,6 @@ import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; import java.util.*; import static ru.spcex.clearing.imdg.base.ObjectBaseMapStore.MAX_IN_CLAUSE_SIZE; @@ -76,14 +75,16 @@ public abstract class AbstractSliceMapLoader implements Autoco @Override public Iterable loadAllKeys() { - log.debug("loading all keys from {}...", getTableName()); - Instant yesterdayInstant = Instant.now().minus(0, ChronoUnit.DAYS); - Instant halfYearAgo = yesterdayInstant.minus(365, ChronoUnit.DAYS); - List ids = jdbcTemplate.query("select id from " + getTableName() + " where CAST(tradingday as DATE) <= ? and CAST(tradingday as DATE) >= ? ", - new Object[]{FIREBIRD_INSTANT_FORMATTER.format(yesterdayInstant), FIREBIRD_INSTANT_FORMATTER.format(halfYearAgo)}, - (resultSet, i) -> resultSet.getObject("id", Long.class)); - log.debug("loading all keys from {} done; size={}", getTableName(), ids.size()); - return ids; + //fixme понять какое поле подходит большинству таблиц. created_at??? + return Collections.emptyList(); +// log.debug("loading all keys from {}...", getTableName()); +// Instant yesterdayInstant = Instant.now().minus(0, ChronoUnit.DAYS); +// Instant halfYearAgo = yesterdayInstant.minus(365, ChronoUnit.DAYS); +// List ids = jdbcTemplate.query("select id from " + getTableName() + " where CAST(tradingday as DATE) <= ? and CAST(tradingday as DATE) >= ? ", +// new Object[]{FIREBIRD_INSTANT_FORMATTER.format(yesterdayInstant), FIREBIRD_INSTANT_FORMATTER.format(halfYearAgo)}, +// (resultSet, i) -> resultSet.getObject("id", Long.class)); +// log.debug("loading all keys from {} done; size={}", getTableName(), ids.size()); +// return ids; } protected LocalDate getLocalDateFromSqlDate(ResultSet rs, String column) throws SQLException { diff --git a/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/SearchMoneyBalanceRegisterMapStore.java b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/SearchMoneyBalanceRegisterMapStore.java index 8dbfce9cd..68c929cb3 100644 --- a/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/SearchMoneyBalanceRegisterMapStore.java +++ b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/SearchMoneyBalanceRegisterMapStore.java @@ -5,8 +5,11 @@ import org.springframework.stereotype.Component; import ru.spcex.clearing.historyimdg.index.SearchProxyMoneyBalanceRegister; import ru.spcex.clearing.imdg.IMDGDistributedNames; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; @Component @@ -20,6 +23,30 @@ public class SearchMoneyBalanceRegisterMapStore extends AbstractSliceMapLoader loadAllKeys() { + //пример loadAllKeys + log.debug("loading all keys from {}...", getTableName()); + LocalDate upperInstant = LocalDate.now().plus(1, ChronoUnit.DAYS); + LocalDate halfYearAgo = upperInstant.minus(365, ChronoUnit.DAYS); + List ids = jdbcTemplate.query("select id from " + getTableName() + " where CAST(created_at as DATE) <= ? and CAST(created_at as DATE) >= ? ", + new Object[]{upperInstant, halfYearAgo}, + (resultSet, i) -> resultSet.getObject("id", Long.class)); + log.debug("loading all keys from {} done; size={}", getTableName(), ids.size()); + return ids; + //LocalDate today = LocalDate.now(); + // log.debug("loadAllKeys from {} on {} {} today", getTableName(), dateField, canBeGreat ? ">=" : "="); + // List keys; + // try { + // keys = jdbcTemplate.query("select id from " + getTableName() + " where " + dateField + (canBeGreat ? " >= ?" : " = ?"), + // (resultSet, i) -> resultSet.getLong("id"), + // new Object[]{today}); + // } catch (Throwable e) { + // log.error("At load keys from {} (by field {}): {}", getTableName(), dateField, ExceptionUtils.getStackTrace(e)); + // throw e; + // } + } + @Override public Collection load(Collection keys) { Map> paramMap = Collections.singletonMap("ids", keys); diff --git a/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/services/HazelcastLifecycleSupport.java b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/services/HazelcastLifecycleSupport.java new file mode 100644 index 000000000..24d55cdec --- /dev/null +++ b/clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/services/HazelcastLifecycleSupport.java @@ -0,0 +1,111 @@ +package ru.spcex.clearing.historyimdg.services; + +import com.hazelcast.core.HazelcastInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import ru.spcex.platform.imdg.iml.hazelcast.util.HazelcastHelper; + +import java.util.Collection; + +@Service +public class HazelcastLifecycleSupport implements InitializingBean, DisposableBean { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final HazelcastInstance hazelcastServerInstance; + private final Collection historyMapNames; + + @Autowired + public HazelcastLifecycleSupport(@Qualifier("hazelcastInstanceImdg") HazelcastInstance hazelcastServerInstance, Collection historyMapNames) { + this.hazelcastServerInstance = hazelcastServerInstance; + this.historyMapNames = historyMapNames; + } + + @Override + public void afterPropertiesSet() throws Exception { + long loadTime = System.currentTimeMillis(); + log.debug("init maps started"); + for (String mapName : hazelcastServerInstance.getConfig().getMapConfigs().keySet()) { + //зачем historyMapNames - изначально чтобы соотв. MapStore не делал loadAllKeys + //по факту в Histry мапсторах метод loadAllKeys должен быть всегда переопределен в emptyList() + //можно потом убрать + if (historyMapNames.contains(mapName)) { + continue; + } + log.debug("configured '{}' map for hazelcast search server. size: {}", mapName, hazelcastServerInstance.getMap(mapName).size()); + } + loadTime = System.currentTimeMillis() - loadTime; + log.info("All map load time {} ms", loadTime); + HazelcastHelper.imdgSystem_setStorageState(true, hazelcastServerInstance); + } + + @Override + public void destroy() throws Exception { + + } + //package com.moex.corp.search.service; + // + //import com.hazelcast.core.HazelcastInstance; + //import com.moex.platform.errors.TextErrorService; + //import com.moex.platform.hazelcast.HazelcastCommon; + //import org.slf4j.Logger; + //import org.slf4j.LoggerFactory; + //import org.springframework.beans.factory.DisposableBean; + //import org.springframework.beans.factory.InitializingBean; + //import org.springframework.beans.factory.annotation.Autowired; + //import org.springframework.beans.factory.annotation.Qualifier; + //import org.springframework.stereotype.Service; + // + //import java.util.Collection; + // + //@SuppressWarnings("Duplicates") + //@Service + //public class HazelcastLifecycleSupport implements InitializingBean, DisposableBean { + // /** + // * Рабочая версия БД. Требуется вручную сверять с DDL.sql и накручивать эту переменную. + // */ + // public static final String CHECK_DB_VERSION = "2.39"; + // + // private final Logger log = LoggerFactory.getLogger(this.getClass()); + // + // private final HazelcastInstance hazelcastServerInstance; + // private final Collection historyMapNames; + // + // @Autowired + // public HazelcastLifecycleSupport(HazelcastInstance hazelcastServerInstance, + // @Qualifier("historyMapNames") Collection historyMapNames) { + // this.hazelcastServerInstance = hazelcastServerInstance; + // this.historyMapNames = historyMapNames; + // } + // + // @Override + // public void afterPropertiesSet() { + // long loadTime = System.currentTimeMillis(); + // log.debug("init maps started"); + // for (String mapName : hazelcastServerInstance.getConfig().getMapConfigs().keySet()) { + // if (historyMapNames.contains(mapName)) { + // continue; + // } + // log.debug("configured '{}' map for hazelcast search server. size: {}", mapName, hazelcastServerInstance.getMap(mapName).size()); + // } + // loadTime = System.currentTimeMillis() - loadTime; + // log.info("All map load time {} ms", loadTime); + // HazelcastCommon.otcSystem_setStorageState(true, hazelcastServerInstance); + // + // TextErrorService.setHazelcast(hazelcastServerInstance); + // + // } + // + // @Override + // public void destroy() { + // hazelcastServerInstance.shutdown(); + // } + // + // public HazelcastInstance getHazelcastServerInstance() { + // return hazelcastServerInstance; + // } + //} +} diff --git a/clearing-parent/imdg-hist/src/main/resources/application.properties b/clearing-parent/imdg-hist/src/main/resources/application.properties index cc380fc77..87fc068e3 100644 --- a/clearing-parent/imdg-hist/src/main/resources/application.properties +++ b/clearing-parent/imdg-hist/src/main/resources/application.properties @@ -1,6 +1,6 @@ -imdg.hist.hazelcast.listenPort=5701 -imdg.hist.hazelcast.login=dev -imdg.hist.hazelcast.password=dev-pass +imdg.hist.hazelcast.listenPort=5702 +imdg.hist.hazelcast.login=dev-hist +imdg.hist.hazelcast.password=dev-pass-hist imdg.hist.hazelcast.cluster-members[0]=127.0.0.1 imdg.hist.database.login=clearing imdg.hist.database.password=Aa111111 diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java index 1722b07a8..195a9db37 100644 --- a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java @@ -232,6 +232,12 @@ public class ImdgHazelcast implements Imdg { return result; } + @Override + public Collection getCollectionIdsByPredicate(ImdgPredicate prdct) { + Predicate hazelcastPredicate = ((ImdgPredicateHazelcast) prdct).getRawPredicate(); + return new ArrayList<>(map.keySet(hazelcastPredicate)); + } + @Override public Long nextIDSequenceFor() { return idGenerator.newId(); diff --git a/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java b/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java index 9beb042df..eb8e6c182 100644 --- a/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java +++ b/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java @@ -64,6 +64,10 @@ public interface Imdg { throw new UnsupportedOperationException("not implemented getCollectionIdsBySQL"); } + default Collection getCollectionIdsByPredicate(ImdgPredicate prdct) { + throw new UnsupportedOperationException("not implemented getCollectionIdsByPredicate"); + } + default Collection getAllValues() { throw new UnsupportedOperationException("not implemented getAllValues"); } diff --git a/z-distr/pom.xml b/z-distr/pom.xml index a0e3395ad..1289682bf 100644 --- a/z-distr/pom.xml +++ b/z-distr/pom.xml @@ -147,7 +147,7 @@ - ${folder_root_clearing_imdg}/target/imdg.jar + ${folder_root_clearing_imdg}/target/imdg-exec.jar ${folder.clearing.distr.bin}/imdg.jar