From 1d33e0cd3b359c7d705769d800643dc8a55cc015 Mon Sep 17 00:00:00 2001 From: ialbert Date: Tue, 9 Aug 2022 20:53:33 +0300 Subject: [PATCH] http://jira.mfd.msk:8088/browse/CLS-11 --- clearing-parent/backend-api/pom.xml | 4 + .../config/BackEndApiImdgConfig.java | 47 ++++ .../backendapi/config/HazelcastSettings.java | 12 + .../src/main/resources/application.properties | 3 + clearing-parent/imdg/pom.xml | 14 + .../imdg/config/HazelcastConfiguration.java | 6 +- .../imdg/config/HazelcastServerElement.java | 43 +++ .../clearing/imdg/config/ImdgSettings.java | 21 ++ .../AbstractHazelcastLifecycleSupport.java | 3 +- .../src/main/resources/application.properties | 1 + .../platform-imdg-api-hazelcast-impl/pom.xml | 65 +++++ .../iml/hazelcast/adapter/ImdgHazelcast.java | 19 ++ .../config/HazelcastClientParams.java | 60 ++++ .../hazelcast/service/HazelcastService.java | 37 +++ .../service/HazelcastServiceBase.java | 259 ++++++++++++++++++ .../service/IHazelcastClusterStatus.java | 10 + .../iml/hazelcast/util/HazelcastHelper.java | 130 +++++++++ .../java/ru/spcex/platform/imdg/api/Imdg.java | 47 ++++ .../spcex/platform/imdg/api/ImdgProvider.java | 11 + .../ru/spcex/platform/imdg/api/Storage.java | 27 -- platform-parent/platform-utils/pom.xml | 22 ++ .../platform/utils/log/ExceptionUtils.java | 13 + platform-parent/pom.xml | 2 + pom.xml | 21 ++ z-distr/pom.xml | 23 ++ 25 files changed, 870 insertions(+), 30 deletions(-) create mode 100644 clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java create mode 100644 clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/HazelcastSettings.java create mode 100644 clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastServerElement.java create mode 100644 clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/ImdgSettings.java create mode 100644 clearing-parent/imdg/src/main/resources/application.properties create mode 100644 platform-parent/platform-imdg-api-hazelcast-impl/pom.xml create mode 100644 platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java create mode 100644 platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/config/HazelcastClientParams.java create mode 100644 platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastService.java create mode 100644 platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastServiceBase.java create mode 100644 platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/IHazelcastClusterStatus.java create mode 100644 platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/util/HazelcastHelper.java create mode 100644 platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java create mode 100644 platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/ImdgProvider.java delete mode 100644 platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Storage.java create mode 100644 platform-parent/platform-utils/pom.xml create mode 100644 platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/log/ExceptionUtils.java diff --git a/clearing-parent/backend-api/pom.xml b/clearing-parent/backend-api/pom.xml index 4ce3d5a40..0fe55529a 100644 --- a/clearing-parent/backend-api/pom.xml +++ b/clearing-parent/backend-api/pom.xml @@ -33,6 +33,10 @@ keycloak-spring-boot-starter ${keycloak-spring-boot-starter.version} + + ru.spcex.platform + platform-imdg-api-hazelcast-impl + diff --git a/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java new file mode 100644 index 000000000..7eb81b6ac --- /dev/null +++ b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java @@ -0,0 +1,47 @@ +package ru.spcex.clearing.backendapi.config; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import ru.spcex.platform.imdg.api.ImdgProvider; +import ru.spcex.platform.imdg.iml.hazelcast.service.HazelcastService; + +@Configuration +public class BackEndApiImdgConfig { + @Bean(name = "taskExecutorHazelcastClientInitializer") + public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer() { + return createThreadPoolTaskExecutor(1, true); + } + + @Bean(name = "taskExecutorIdGeneratorAwaiter") + public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter() { + return createThreadPoolTaskExecutor(1, false); + } + + @Autowired + @Bean + public ImdgProvider imdgProvider( + @Qualifier("taskExecutorHazelcastClientInitializer") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + @Qualifier("taskExecutorIdGeneratorAwaiter") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + HazelcastSettings clientSetting + ) { + return new HazelcastService(taskExecutorHazelcastClientInitializer, + taskExecutorIdGeneratorAwaiter, + clientSetting); + } + + + private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) { + ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor(); + if (maxPoolSz > 2) { + pool.setKeepAliveSeconds(60); + pool.setAllowCoreThreadTimeOut(true); + } + pool.setCorePoolSize(maxPoolSz); + pool.setWaitForTasksToCompleteOnShutdown(waitForCompletion); + return pool; + } + +} diff --git a/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/HazelcastSettings.java b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/HazelcastSettings.java new file mode 100644 index 000000000..d6e5de56c --- /dev/null +++ b/clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/HazelcastSettings.java @@ -0,0 +1,12 @@ +package ru.spcex.clearing.backendapi.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; +import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams; + +@Component +@PropertySource("file:${spring.config.location}/application.properties") +@ConfigurationProperties("backend-api.hazelcast") +public class HazelcastSettings extends HazelcastClientParams { +} diff --git a/clearing-parent/backend-api/src/main/resources/application.properties b/clearing-parent/backend-api/src/main/resources/application.properties index b297aa7f8..e21d692d6 100644 --- a/clearing-parent/backend-api/src/main/resources/application.properties +++ b/clearing-parent/backend-api/src/main/resources/application.properties @@ -3,6 +3,9 @@ server.servlet.context-path=/backend-api spring.main.web-application-type=servlet backend-api.example-setting=test +backend-api.hazelcast.cluster-members=127.0.0.1 +backend-api.hazelcast.login=dev +backend-api.hazelcast.password=dev-pass ##keycloak diff --git a/clearing-parent/imdg/pom.xml b/clearing-parent/imdg/pom.xml index 2d54dc66f..f465c5845 100644 --- a/clearing-parent/imdg/pom.xml +++ b/clearing-parent/imdg/pom.xml @@ -64,10 +64,24 @@ hazelcast-all ${external_libraries.hazelcast.version} + + ru.spcex.platform + platform-imdg-api-hazelcast-impl + jar/${project.artifactId} + + + src/main/resources + + application.properties + + false + + + org.apache.maven.plugins diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastConfiguration.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastConfiguration.java index ab36d0ad8..83ac8391d 100644 --- a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastConfiguration.java +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastConfiguration.java @@ -12,10 +12,12 @@ public class HazelcastConfiguration { private ConfigurationRootElement configRoot = DfaConfig.get().getRoot(); private final PoolMapConfigs poolMapConfigs; + private final HazelcastServerElement hzSettings; @Autowired - public HazelcastConfiguration(PoolMapConfigs poolMapConfigs) { + public HazelcastConfiguration(PoolMapConfigs poolMapConfigs, ImdgSettings imdgSettings) { this.poolMapConfigs = poolMapConfigs; + this.hzSettings = imdgSettings.getHazelcast(); } @Bean @@ -37,7 +39,7 @@ public class HazelcastConfiguration { .setMulticastConfig(new MulticastConfig() .setEnabled(false)) .setTcpIpConfig(new TcpIpConfig() - .setEnabled(true) + .setEnabled(true).setMembers(hzSettings.getClusterMembers()) ) ) ); diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastServerElement.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastServerElement.java new file mode 100644 index 000000000..db267c74e --- /dev/null +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/HazelcastServerElement.java @@ -0,0 +1,43 @@ +package ru.spcex.clearing.imdg.config; + +import java.util.List; + +public class HazelcastServerElement { + + private int listenPort = 5071; + private String login = "dev"; + private String password = "dev-pass"; + private List clusterMembers; + + public int getListenPort() { + return listenPort; + } + + public void setListenPort(int listenPort) { + this.listenPort = listenPort; + } + + public String getLogin() { + return login; + } + + public void setLogin(String login) { + this.login = login; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public List getClusterMembers() { + return clusterMembers; + } + + public void setClusterMembers(List clusterMembers) { + this.clusterMembers = clusterMembers; + } +} diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/ImdgSettings.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/ImdgSettings.java new file mode 100644 index 000000000..9a0769932 --- /dev/null +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/ImdgSettings.java @@ -0,0 +1,21 @@ +package ru.spcex.clearing.imdg.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; + +@Component +@PropertySource("file:${spring.config.location}/application.properties") +@ConfigurationProperties("imdg") +public class ImdgSettings { + + private HazelcastServerElement hazelcast; + + public HazelcastServerElement getHazelcast() { + return hazelcast; + } + + public void setHazelcast(HazelcastServerElement hazelcast) { + this.hazelcast = hazelcast; + } +} diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java index 677dc5367..fcd08aade 100644 --- a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java @@ -15,6 +15,7 @@ import ru.clearing.classes.objects.BusinessObject; import ru.spcex.clearing.imdg.base.DictionaryMapStore; import ru.spcex.clearing.imdg.base.SimpleObjectMapStore; import ru.spcex.clearing.imdg.utils.IMDGDistributedNames; +import ru.spcex.platform.imdg.iml.hazelcast.util.HazelcastHelper; import java.util.ArrayList; import java.util.List; @@ -125,7 +126,7 @@ public abstract class AbstractHazelcastLifecycleSupport implements InitializingB throw new RuntimeException("MapStore multithreaded not complete.", e); } -// HazelcastCommon.otcSystem_setStorageState(true, hazelcastServerInstance); todo в других модулях может быть проверка на это, и должна быть. + HazelcastHelper.otcSystem_setStorageState(true, hazelcastServerInstance); // hazelcastServerInstance.getClientService().addClientListener(clientListener); loadTime = System.currentTimeMillis() - loadTime; log.info("All map load time {} ms", loadTime); diff --git a/clearing-parent/imdg/src/main/resources/application.properties b/clearing-parent/imdg/src/main/resources/application.properties new file mode 100644 index 000000000..f910e21fe --- /dev/null +++ b/clearing-parent/imdg/src/main/resources/application.properties @@ -0,0 +1 @@ +imdg.hazelcast.cluster-members[0]=127.0.0.1 \ No newline at end of file diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/pom.xml b/platform-parent/platform-imdg-api-hazelcast-impl/pom.xml new file mode 100644 index 000000000..d971e2b7e --- /dev/null +++ b/platform-parent/platform-imdg-api-hazelcast-impl/pom.xml @@ -0,0 +1,65 @@ + + + + platform-parent + ru.spcex.platform + 1.0.0 + + 4.0.0 + + platform-imdg-api-hazelcast-impl + 1.0.0 + + + 17 + 17 + + + + + ru.spcex.platform + platform-imdg-api + + + ru.spcex.platform + platform-utils + + + com.hazelcast + hazelcast-all + ${external_libraries.hazelcast.version} + + + + ch.qos.logback + logback-classic + ${external_libraries.logback.version} + provided + + + ch.qos.logback + logback-core + ${external_libraries.logback.version} + provided + + + org.slf4j + slf4j-api + ${external_libraries.slf4j.version} + provided + + + org.springframework + spring-core + provided + + + org.springframework + spring-context + provided + + + + \ No newline at end of file diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java new file mode 100644 index 000000000..34b201a88 --- /dev/null +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java @@ -0,0 +1,19 @@ +package ru.spcex.platform.imdg.iml.hazelcast.adapter; + +import com.hazelcast.core.IMap; +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.imdg.api.Imdg; + +public class ImdgHazelcast implements Imdg { + + private IMap map; + + public IMap getMap() { + return map; + } + + public void setMap(IMap map) { + this.map = map; + } + //todo implement methods +} diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/config/HazelcastClientParams.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/config/HazelcastClientParams.java new file mode 100644 index 000000000..5b2c8f83b --- /dev/null +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/config/HazelcastClientParams.java @@ -0,0 +1,60 @@ +package ru.spcex.platform.imdg.iml.hazelcast.config; + +import com.hazelcast.config.NearCacheConfig; + +@SuppressWarnings({"FieldCanBeLocal", "unused"}) +public class HazelcastClientParams { + + public HazelcastClientParams() { + } + + private String clusterMembers; + + private String login; + + private String password; + + private String instanceName; + + private NearCacheConfig nearCacheConfig; + + public String getClusterMembers() { + return clusterMembers; + } + + public void setClusterMembers(String clusterMembers) { + this.clusterMembers = clusterMembers; + } + + public String getLogin() { + return login; + } + + public void setLogin(String login) { + this.login = login; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getInstanceName() { + return instanceName; + } + + public void setInstanceName(String instanceName) { + this.instanceName = instanceName; + } + + public NearCacheConfig getNearCacheConfig() { + return nearCacheConfig; + } + + public void setNearCacheConfig(NearCacheConfig nearCacheConfig) { + this.nearCacheConfig = nearCacheConfig; + } +} diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastService.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastService.java new file mode 100644 index 000000000..a19d1d517 --- /dev/null +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastService.java @@ -0,0 +1,37 @@ +package ru.spcex.platform.imdg.iml.hazelcast.service; + + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams; + +public final class HazelcastService extends HazelcastServiceBase implements InitializingBean, DisposableBean { + + public HazelcastService(ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + HazelcastClientParams hazelcastClientParams) { + super(taskExecutorHazelcastClientInitializer, taskExecutorIdGeneratorAwaiter, hazelcastClientParams); + } + + public void init() { + taskExecutorHazelcastClientInitializer.submit(this::reinitializeHazelcastClient); + } + + @Scheduled(cron = "* 0 0 * * *") + @Override + protected void reloadClient() { + super.reloadClient(); + } + + @Override + public void destroy() { + shutdown(); + } + + @Override + public void afterPropertiesSet() throws Exception { + init(); + } +} diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastServiceBase.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastServiceBase.java new file mode 100644 index 000000000..93c026724 --- /dev/null +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/HazelcastServiceBase.java @@ -0,0 +1,259 @@ +package ru.spcex.platform.imdg.iml.hazelcast.service; + +import com.hazelcast.client.HazelcastClient; +import com.hazelcast.client.config.ClientConfig; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.core.LifecycleEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import ru.spcex.platform.classes.base.SpcexObjectBase; +import ru.spcex.platform.imdg.api.Imdg; +import ru.spcex.platform.imdg.api.ImdgProvider; +import ru.spcex.platform.imdg.iml.hazelcast.adapter.ImdgHazelcast; +import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams; +import ru.spcex.platform.imdg.iml.hazelcast.util.HazelcastHelper; +import ru.spcex.platform.utils.log.ExceptionUtils; + +import java.util.*; + +public abstract class HazelcastServiceBase +// implements IHazelcastService + implements ImdgProvider +{ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + public final int SLEEP_AFTER_RELOAD_MS = 5000; + protected String mapNameForCache; + protected final ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer; + protected final ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter; + protected final HazelcastClientParams settingsElementHazelcastClient; + + protected final Collection hazelcastStatusSubscribers = new ArrayList<>(); + protected HazelcastInstance hazelcastInstance; + protected volatile boolean isAvailableNow; + private Environment environment; + + public HazelcastServiceBase( + ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + HazelcastClientParams settingsElementHazelcastClient, + Environment environment) { + this.taskExecutorIdGeneratorAwaiter = taskExecutorIdGeneratorAwaiter; + this.taskExecutorHazelcastClientInitializer = taskExecutorHazelcastClientInitializer; + this.settingsElementHazelcastClient = settingsElementHazelcastClient; + this.environment = environment; + } + + public HazelcastServiceBase( + ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer, + ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter, + HazelcastClientParams settingsElementHazelcastClient) { + this.taskExecutorIdGeneratorAwaiter = taskExecutorIdGeneratorAwaiter; + this.taskExecutorHazelcastClientInitializer = taskExecutorHazelcastClientInitializer; + this.settingsElementHazelcastClient = settingsElementHazelcastClient; + } + + private void createHazelcastClient() { + if (hazelcastInstance != null) { + try { + hazelcastInstance.shutdown(); + } catch (Throwable ignored) { + } + } + log.info("Hazelcast: client init"); + ClientConfig clientConfig = HazelcastHelper.getClientConfig( + settingsElementHazelcastClient.getClusterMembers(), + settingsElementHazelcastClient.getLogin(), + settingsElementHazelcastClient.getPassword(), + settingsElementHazelcastClient.getInstanceName(), + settingsElementHazelcastClient.getNearCacheConfig() + ); + if (environment != null && Arrays.asList(environment.getActiveProfiles()).contains("tests")) { + clientConfig.getNetworkConfig().setConnectionAttemptLimit(HazelcastHelper.TEST_CONNECTION_ATTEMPT_LIMIT); + } + log.info("Hazelcast: client created, trying connect to server ({}) ...", settingsElementHazelcastClient.getClusterMembers()); + hazelcastInstance = HazelcastClient.newHazelcastClient(clientConfig); + log.info("Hazelcast: client created and connected"); + } + +// @Override + public void reinitializeHazelcastClient() { + try { + createHazelcastClient(); + Thread.sleep(100); // await for skip CLIENT_CONNECTED event. See warn "Hazelcast already available, but has call onAvailable() again." + awaitGeneratorId(); + hazelcastInstance.getLifecycleService().addLifecycleListener((LifecycleEvent event) -> { + switch (event.getState()) { + case CLIENT_CONNECTED: + log.info("Hazelcast: event connected"); + awaitGeneratorId(); + break; + case CLIENT_DISCONNECTED: + log.info("Hazelcast: event disconnected"); + onUnavailable(); + break; + } + }); + } catch (Throwable e) { + log.error("reinitializeHazelcastClient() has error: {}", ExceptionUtils.getStackTrace(e)); + } + } + + protected void onAvailable() { + if (isAvailableNow) { + log.warn("Hazelcast already available, but has call onAvailable() again. Ignore."); + return; // пропустить + } + isAvailableNow = false; + synchronized (hazelcastStatusSubscribers) { + hazelcastStatusSubscribers.forEach(hazelcastStatusSubscriber -> { + log.trace("hazelcastStatusSubscribers call> {}", hazelcastStatusSubscriber); + try { + hazelcastStatusSubscriber.getAvailable(hazelcastInstance); + } catch (Throwable e) { + log.warn(ExceptionUtils.getStackTrace(e)); + } + }); + isAvailableNow = true; + } + } + + protected void onUnavailable() { + if (!isAvailableNow) + log.warn("Hazelcast already unavailable, but has call onUnavailable() again."); + isAvailableNow = false; + synchronized (hazelcastStatusSubscribers) { + hazelcastStatusSubscribers.forEach(hazelcastStatusSubscriber -> { + log.trace("hazelcastStatusSubscribers call unavailable> {}", hazelcastStatusSubscriber); + try { + hazelcastStatusSubscriber.getUnavailable(hazelcastInstance); + } catch (Throwable e) { + log.warn(ExceptionUtils.getStackTrace(e)); + } + }); + if (isAvailableNow) { // never: если успел проскочить между isAvailableNow = false и synchronized + log.warn("Concurrent call onAvailable() and onUnavailable(). Stacktrace: {}", ExceptionUtils.getStackTrace(new ConcurrentModificationException("Warning"))); + isAvailableNow = false; + } + } + } + + protected void reloadClient() { + String logMsg = "Reloading Hazelcast client:"; + Date dtStart = new Date(); + try { + onUnavailable(); + Thread.sleep(SLEEP_AFTER_RELOAD_MS); + reinitializeHazelcastClient(); + logMsg = String.format("%s done OK! (duration=%dms)", + logMsg, (new Date()).getTime() - dtStart.getTime()); + } catch (Throwable e) { + logMsg = String.format("%s error (duration=%dms):\n%s", + logMsg, (new Date()).getTime() - dtStart.getTime(), ExceptionUtils.getStackTrace(e)); + } finally { + log.info(logMsg); + } + } + + protected void awaitGeneratorId() { + taskExecutorIdGeneratorAwaiter.execute(() -> { + try { + // ожидание инициализации Storage (только после инициализации IDGenerator начинать работу). + boolean done = false; + while (!done) { + log.debug("Execute awaitGeneratorId()"); + try { + HazelcastHelper.otcSystem_waitTillReadyState(getHazelcast()); + onAvailable(); + done = true; + } catch (Throwable e) { + log.info(String.format("Waiting Hazelcast: %s -> %s", e.getClass().getSimpleName(), e.getMessage())); + createHazelcastClient(); + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + log.warn("awaitGeneratorId thread interrupted!"); + Thread.currentThread().interrupt(); + return; + } + } + } + // проверка версии классов +// HazelcastCommon.verifyVersionClientsClasses(hazelcastInstance); + } catch (Throwable e) { + log.error("Error at awaitGeneratorId(): {}", ExceptionUtils.getStackTrace(e)); + } + }); + } + +// @Override + public void statusSubscribe(IHazelcastClusterStatus clusterStatus) { + log.trace("hazelcastStatusSubscribers add> {}", clusterStatus); + synchronized (hazelcastStatusSubscribers) { + if (hazelcastStatusSubscribers.contains(clusterStatus)) // never + throw new IllegalStateException("ClusterStatus listener " + clusterStatus + " already added!"); + hazelcastStatusSubscribers.add(clusterStatus); + if (isAvailableNow) { // register now + log.trace("hazelcastStatusSubscribers added and call now> {}", clusterStatus); + try { + clusterStatus.getAvailable(hazelcastInstance); + } catch (Throwable e) { + log.warn(ExceptionUtils.getStackTrace(e)); + } + } + } + } + +// @Override + public HazelcastInstance getHazelcast() { + return hazelcastInstance; + } + + @Override + public Imdg getImdg(String key, Class clazz) { + ImdgHazelcast imdg = new ImdgHazelcast<>(); + imdg.setMap(getHazelcast().getMap(key)); + return imdg; + } + + // @Override + public void waitTillReadyState() throws InterruptedException { + boolean done = false; + int sleepCount = 0; + while (!done) { + try { + HazelcastInstance instance = getHazelcast(); + if (instance != null) { + HazelcastHelper.otcSystem_waitTillReadyState(instance); + done = true; + break; + } else { + if (sleepCount++ % 30 == 0) + log.debug("Hazelcast not connected. Wait till STORAGE be ready..."); + } + } catch (InterruptedException e) { + log.warn(String.format("Waiting Hazelcast interrupted: %s -> %s", e.getClass().getSimpleName(), e.getMessage())); + throw e; + } + Thread.sleep(100); + } + } + +// @Override + public void shutdown() { + isAvailableNow = false; + if (hazelcastInstance != null) + hazelcastInstance.shutdown(); + } + + public String getMapNameForCache() { + return mapNameForCache; + } + + public void setMapNameForCache(String mapNameForCache) { + this.mapNameForCache = mapNameForCache; + } +} + diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/IHazelcastClusterStatus.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/IHazelcastClusterStatus.java new file mode 100644 index 000000000..258430164 --- /dev/null +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/service/IHazelcastClusterStatus.java @@ -0,0 +1,10 @@ +package ru.spcex.platform.imdg.iml.hazelcast.service; + +import com.hazelcast.core.HazelcastInstance; + +public interface IHazelcastClusterStatus { + void getAvailable(HazelcastInstance hazelcastNotInited); + + void getUnavailable(HazelcastInstance hazelcastNotInited); + +} diff --git a/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/util/HazelcastHelper.java b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/util/HazelcastHelper.java new file mode 100644 index 000000000..14b9f3394 --- /dev/null +++ b/platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/util/HazelcastHelper.java @@ -0,0 +1,130 @@ +package ru.spcex.platform.imdg.iml.hazelcast.util; + +import com.hazelcast.client.HazelcastClient; +import com.hazelcast.client.config.ClientConfig; +import com.hazelcast.client.config.ClientNetworkConfig; +import com.hazelcast.config.NearCacheConfig; +import com.hazelcast.core.HazelcastException; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.core.IMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class HazelcastHelper { + private static final Logger log = LoggerFactory.getLogger(HazelcastHelper.class); + + public static final int DEFAULT_CONNECTION_TIMEOUT_SEC = 30; + public static final int DEFAULT_CONNECTION_ATTEMPT_PERIOD_SEC = 10; + public static final int DEFAULT_CONNECTION_ATTEMPT_LIMIT = Integer.MAX_VALUE; + public static final int TEST_CONNECTION_ATTEMPT_LIMIT = 1; + public static final String OTC_SYSTEM_MAP = "OTC_SYSTEM"; + + public static ClientConfig getClientConfig(String members, String login, String password, String instanceName) { + return getClientConfig(members, login, password, instanceName, null); + } + + public static ClientConfig getClientConfig(String members, String login, String password, String instanceName, NearCacheConfig nearCacheConfig) { + if (members == null) { + throw new IllegalArgumentException("members must not be null!"); + } + String[] mm = members.split(","); + if (mm.length == 0) { + throw new IllegalArgumentException("members must not be empty!"); + } + ClientNetworkConfig clientNetworkConfig = new ClientNetworkConfig(); + for (String m : mm) { + if (m != null && !m.isEmpty()) { + clientNetworkConfig.addAddress(m); + } + } + ClientConfig config = new ClientConfig(); + config.setInstanceName(instanceName); + config.setNetworkConfig(clientNetworkConfig); + + config.setProperty("hazelcast.logging.type", "slf4j"); + + config.getGroupConfig().setName(login); + config.getGroupConfig().setPassword(password); + + config.getNetworkConfig().setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT_SEC * 1000); + config.getNetworkConfig().setConnectionAttemptPeriod(DEFAULT_CONNECTION_ATTEMPT_PERIOD_SEC * 1000); + config.getNetworkConfig().setConnectionAttemptLimit(DEFAULT_CONNECTION_ATTEMPT_LIMIT); + + if (nearCacheConfig != null) config.addNearCacheConfig(nearCacheConfig); + return config; + } + + + /** + * Создаёт клиента HazelcastInstance, подключённого по настрйокам ClientConfig. + * Ожидает готовности Storage, инициализирует TextErrorService. + * В случае ошибок делает повторение попыток создания клиента. + *

+ * Пример получения ClientConfig: + * + * HazelcastHelper.getClientConfig( + * Config.get().getHazelcastSettings().getClusterMembers(), + * Config.get().getHazelcastSettings().getLogin(), + * Config.get().getHazelcastSettings().getPassword() + * ) + * + * + * @param hzClientConfig + * @return HazelcastClient + */ + public HazelcastInstance makeHazelcastClientAndWaitTillReady(ClientConfig hzClientConfig) { + HazelcastInstance instance = null; + // ожидание инициализации Storage (только после инициализации IDGenerator начинать работу). + boolean done = false; + while (!done) { + try { + instance = HazelcastClient.newHazelcastClient(hzClientConfig);// createHzClient(); + otcSystem_waitTillReadyState(instance); +// TextErrorService.setHazelcast(instance); + log.info("Hazelcast: TextErrorService done"); + done = true; + } catch (InterruptedException e) { + throw new RuntimeException("Thread interrupted when waiting storage ready.", e); + } catch (Exception e) { + log.info(String.format("Waiting Hazelcast: %s -> %s", e.getClass().getSimpleName(), e.getMessage())); + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + throw new RuntimeException("Thread interrupted.", e); + } + } + } + // проверка версии классов +// HazelcastCommon.verifyVersionClientsClasses(instance); + return instance; + } + + public static void otcSystem_waitTillReadyState(HazelcastInstance hazelcast) throws InterruptedException { + int sleepCount = 0; + try { + while (!otcSystem_getStorageState(hazelcast)) { + if (sleepCount++ % 30 == 0) + log.info("Wait till STORAGE be ready..."); + Thread.sleep(100); + } + } catch (HazelcastException hcEx) { + // проброс InterruptException из Hazelcast + if (hcEx instanceof HazelcastException && hcEx.getCause() instanceof InterruptedException) + throw new InterruptedException("Can not obtain storage for check state, cause has interrupted. " + hcEx); + throw hcEx; + } + } + + public static boolean otcSystem_getStorageState(HazelcastInstance hazelcast) { + IMap systemMap = hazelcast.getMap(OTC_SYSTEM_MAP); + return Boolean.valueOf(systemMap.get("STORAGE.STATE")); + } + + public static void otcSystem_setStorageState(boolean val, HazelcastInstance hazelcast) { + IMap systemMap = hazelcast.getMap(OTC_SYSTEM_MAP); + systemMap.set("STORAGE.STATE", Boolean.toString(val)); + } + + + +} diff --git a/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java b/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java new file mode 100644 index 000000000..259d607b6 --- /dev/null +++ b/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java @@ -0,0 +1,47 @@ +package ru.spcex.platform.imdg.api; + +import ru.spcex.platform.classes.base.SpcexObjectBase; + +import java.util.Collection; + +public interface Imdg { + default void insert(T paramT) { + throw new UnsupportedOperationException("not implemented insert"); + } + + default void update(T paramT) { + throw new UnsupportedOperationException("not implemented update"); + } + + default void delete(T paramT) { + throw new UnsupportedOperationException("not implemented delete"); + } + + default Long nextIDSequenceFor() { + throw new UnsupportedOperationException("not implemented nextIDSequenceFor"); + } + + default Collection getCollectionObjectsBySQL(String paramString) { + throw new UnsupportedOperationException("not implemented getCollectionObjectsBySQL"); + } + + default T getSingleObjectBySQL(String paramString) { + throw new UnsupportedOperationException("not implemented getSingleObjectBySQL"); + } + + default T getSingleObjectByID(Long paramLong) { + throw new UnsupportedOperationException("not implemented getSingleObjectByID"); + } + + default Collection getCollectionIdsBySQL(String paramString) { + throw new UnsupportedOperationException("not implemented getCollectionIdsBySQL"); + } + + default Collection getAllValues() { + throw new UnsupportedOperationException("not implemented getAllValues"); + } + + default Collection projectionsAttributeBySql(String paramString, String... paramVarArgs) { + throw new UnsupportedOperationException("not implemented projectionsAttributeBySql"); + } +} diff --git a/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/ImdgProvider.java b/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/ImdgProvider.java new file mode 100644 index 000000000..89b1f3f6d --- /dev/null +++ b/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/ImdgProvider.java @@ -0,0 +1,11 @@ +package ru.spcex.platform.imdg.api; + +import ru.spcex.platform.classes.base.SpcexObjectBase; + +public interface ImdgProvider { + /** + * пока простой интерфейс для получения доступа к мапам + */ + public Imdg getImdg(String key, Class clazz); + +} diff --git a/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Storage.java b/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Storage.java deleted file mode 100644 index 8726f045f..000000000 --- a/platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Storage.java +++ /dev/null @@ -1,27 +0,0 @@ -package ru.spcex.platform.imdg.api; - -import ru.spcex.platform.classes.base.SpcexObjectBase; - -import java.util.Collection; - -public interface Storage { - void insert(Class paramClass, T paramT); - - void update(Class paramClass, T paramT); - - void delete(Class paramClass, T paramT); - - Long nextIDSequenceFor(); - - Collection getCollectionObjectsBySQL(Class paramClass, String paramString); - - T getSingleObjectBySQL(Class paramClass, String paramString); - - T getSingleObjectByID(Class paramClass, Long paramLong); - - Collection getCollectionIdsBySQL(Class paramClass, String paramString); - - Collection getAllValues(Class paramClass); - - Collection projectionsAttributeBySql(Class paramClass, String paramString, String... paramVarArgs); -} diff --git a/platform-parent/platform-utils/pom.xml b/platform-parent/platform-utils/pom.xml new file mode 100644 index 000000000..b9e306f5e --- /dev/null +++ b/platform-parent/platform-utils/pom.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + platform-utils + Platform utilities + jar + 1.0.0 + + + platform-parent + ru.spcex.platform + 1.0.0 + + + + 17 + 17 + + + \ No newline at end of file diff --git a/platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/log/ExceptionUtils.java b/platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/log/ExceptionUtils.java new file mode 100644 index 000000000..27bfe411b --- /dev/null +++ b/platform-parent/platform-utils/src/main/java/ru/spcex/platform/utils/log/ExceptionUtils.java @@ -0,0 +1,13 @@ +package ru.spcex.platform.utils.log; + +import java.io.PrintWriter; +import java.io.StringWriter; + +public class ExceptionUtils { + public static String getStackTrace(Throwable throwable) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw, true); + throwable.printStackTrace(pw); + return sw.getBuffer().toString(); + } +} diff --git a/platform-parent/pom.xml b/platform-parent/pom.xml index 83cf3032b..a9dae8e33 100644 --- a/platform-parent/pom.xml +++ b/platform-parent/pom.xml @@ -20,5 +20,7 @@ platform-imdg-api platform-classes-base + platform-imdg-api-hazelcast-impl + platform-utils \ No newline at end of file diff --git a/pom.xml b/pom.xml index fc38f55cd..f7aa5a677 100644 --- a/pom.xml +++ b/pom.xml @@ -31,10 +31,13 @@ ${folder_root_clearing_temp} ${folder_root_clearing}/clearing-parent/backend-api + ${folder_root_clearing}/clearing-parent/imdg ${folder_root_clearing}/clearing-parent/dbf-exporter ${folder_root_clearing}/clearing-parent/dbf-importer 3.12.4 + 1.7.33 + 1.2.10 @@ -52,6 +55,11 @@ backend-api ${global.project.version} + + ru.spcex.clearing + imdg + ${global.project.version} + ru.spcex.clearing classes @@ -73,6 +81,19 @@ platform-classes-base 1.0.0 + + + ru.spcex.platform + platform-imdg-api-hazelcast-impl + 1.0.0 + + + + ru.spcex.platform + platform-utils + 1.0.0 + + ru.spcex.platform platform-imdg-api diff --git a/z-distr/pom.xml b/z-distr/pom.xml index 87d3bd5ea..dc9f8327c 100644 --- a/z-distr/pom.xml +++ b/z-distr/pom.xml @@ -23,6 +23,10 @@ ru.spcex.clearing backend-api + + ru.spcex.clearing + imdg + @@ -106,6 +110,25 @@ + + copy-imdg-bin + prepare-package + + copy + + + + + ${folder_root_clearing_imdg}/target/jar/imdg-exec.jar + ${folder.clearing.distr.modules}/imdg/imdg-exec.jar + + + ${folder_root_clearing_imdg}/src/main/resources/application.properties + ${folder.clearing.distr.modules}/imdg/application.properties + + + + copy-dbf-exporter-bin