treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

imdg-hist example

Browse source
This commit is contained in:
ialbert 2023-10-13 16:35:11 +03:00
parent 7b422acc4a
commit 039f1f8510
11 changed files with 209 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/BackEndApiImdgConfig.java
View file

@ -21,6 +21,16 @@ public class BackEndApiImdgConfig {
return createThreadPoolTaskExecutor(1, false);
}
@Bean(name = "taskExecutorHazelcastClientInitializerHist")
public ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializerHist() {
return createThreadPoolTaskExecutor(1, true);
}
@Bean(name = "taskExecutorIdGeneratorAwaiterHist")
public ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiterHist() {
return createThreadPoolTaskExecutor(1, false);
}
@Autowired
@Bean
public ImdgProvider imdgProvider(
@ -35,6 +45,20 @@ public class BackEndApiImdgConfig {
return imdg;
}
@Autowired
@Bean
public ImdgProvider imdgProviderHist(
@Qualifier("taskExecutorHazelcastClientInitializerHist") ThreadPoolTaskExecutor taskExecutorHazelcastClientInitializer,
@Qualifier("taskExecutorIdGeneratorAwaiterHist") ThreadPoolTaskExecutor taskExecutorIdGeneratorAwaiter,
BackendApiSettings clientSetting
) {
ImdgProvider imdg = new HazelcastService(taskExecutorHazelcastClientInitializer,
taskExecutorIdGeneratorAwaiter,
clientSetting.getHazelcastSearch());
// todo корректное ожидание готовности imdg.waitAvailable();
return imdg;
}
private static ThreadPoolTaskExecutor createThreadPoolTaskExecutor(int maxPoolSz, boolean waitForCompletion) {
ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();

9
clearing-parent/backend-api/src/main/java/ru/spcex/clearing/backendapi/config/element/BackendApiSettings.java
View file

@ -13,6 +13,7 @@ import ru.spcex.platform.imdg.iml.hazelcast.config.HazelcastClientParams;
@ConfigurationProperties("backend-api")
public class BackendApiSettings {
private HazelcastClientParams hazelcast;
private HazelcastClientParams hazelcastSearch;
private KafkaProducerSettings kafkaProducer;
private KafkaConsumerSettings kafkaConsumer;
private SecuritySettings security;
@ -57,4 +58,12 @@ public class BackendApiSettings {
public void setSecurity(SecuritySettings security) {
this.security = security;
}
public HazelcastClientParams getHazelcastSearch() {
return hazelcastSearch;
}
public void setHazelcastSearch(HazelcastClientParams hazelcastSearch) {
this.hazelcastSearch = hazelcastSearch;
}
}

3
clearing-parent/backend-api/src/main/resources/application.properties
View file

@ -9,6 +9,9 @@ backend-api.example-setting=test
backend-api.hazelcast.cluster-members=127.0.0.1:5701
backend-api.hazelcast.login=dev
backend-api.hazelcast.password=dev-pass
backend-api.hazelcast-search.cluster-members=127.0.0.1:5702
backend-api.hazelcast-search.login=dev-hist
backend-api.hazelcast-search.password=dev-pass-hist
backend-api.kafka-producer.bootstrap-servers=localhost:9092
backend-api.kafka-producer.acks=all
backend-api.kafka-producer.retries=0

15
clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/config/HistoryPoolMapConfigs.java
View file

@ -6,14 +6,12 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import ru.spcex.clearing.imdg.base.AutoconfiguredMap;
import ru.spcex.clearing.imdg.base.DictionaryTMapStore;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.*;
@Configuration
public class HistoryPoolMapConfigs {
@ -91,4 +89,13 @@ public class HistoryPoolMapConfigs {
log.debug("Configured {} mapStore's", out.size());
return out;
}
//fixme remove
@Bean("historyMapNames")
public Set<String> historyMaps() {
Set<String> historyMaps = new HashSet<>();
// historyMaps.add(IMDGDistributedNames.Map_MoneyBalanceRegister);
return historyMaps;
}
}

19
clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/AbstractSliceMapLoader.java
View file

@ -16,7 +16,6 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.*;
import static ru.spcex.clearing.imdg.base.ObjectBaseMapStore.MAX_IN_CLAUSE_SIZE;
@ -76,14 +75,16 @@ public abstract class AbstractSliceMapLoader<T extends WithId> implements Autoco
@Override
public Iterable<Long> loadAllKeys() {
log.debug("loading all keys from {}...", getTableName());
Instant yesterdayInstant = Instant.now().minus(0, ChronoUnit.DAYS);
Instant halfYearAgo = yesterdayInstant.minus(365, ChronoUnit.DAYS);
List<Long> ids = jdbcTemplate.query("select id from " + getTableName() + " where CAST(tradingday as DATE) <= ? and CAST(tradingday as DATE) >= ? ",
new Object[]{FIREBIRD_INSTANT_FORMATTER.format(yesterdayInstant), FIREBIRD_INSTANT_FORMATTER.format(halfYearAgo)},
(resultSet, i) -> resultSet.getObject("id", Long.class));
log.debug("loading all keys from {} done; size={}", getTableName(), ids.size());
return ids;
//fixme понять какое поле подходит большинству таблиц. created_at???
return Collections.emptyList();
// log.debug("loading all keys from {}...", getTableName());
// Instant yesterdayInstant = Instant.now().minus(0, ChronoUnit.DAYS);
// Instant halfYearAgo = yesterdayInstant.minus(365, ChronoUnit.DAYS);
// List<Long> ids = jdbcTemplate.query("select id from " + getTableName() + " where CAST(tradingday as DATE) <= ? and CAST(tradingday as DATE) >= ? ",
// new Object[]{FIREBIRD_INSTANT_FORMATTER.format(yesterdayInstant), FIREBIRD_INSTANT_FORMATTER.format(halfYearAgo)},
// (resultSet, i) -> resultSet.getObject("id", Long.class));
// log.debug("loading all keys from {} done; size={}", getTableName(), ids.size());
// return ids;
}
protected LocalDate getLocalDateFromSqlDate(ResultSet rs, String column) throws SQLException {

27
clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/mapstores/SearchMoneyBalanceRegisterMapStore.java
View file

@ -5,8 +5,11 @@ import org.springframework.stereotype.Component;
import ru.spcex.clearing.historyimdg.index.SearchProxyMoneyBalanceRegister;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@Component
@ -20,6 +23,30 @@ public class SearchMoneyBalanceRegisterMapStore extends AbstractSliceMapLoader<S
return "money_balance_register";
}
@Override
public Iterable<Long> loadAllKeys() {
//пример loadAllKeys
log.debug("loading all keys from {}...", getTableName());
LocalDate upperInstant = LocalDate.now().plus(1, ChronoUnit.DAYS);
LocalDate halfYearAgo = upperInstant.minus(365, ChronoUnit.DAYS);
List<Long> ids = jdbcTemplate.query("select id from " + getTableName() + " where CAST(created_at as DATE) <= ? and CAST(created_at as DATE) >= ? ",
new Object[]{upperInstant, halfYearAgo},
(resultSet, i) -> resultSet.getObject("id", Long.class));
log.debug("loading all keys from {} done; size={}", getTableName(), ids.size());
return ids;
//LocalDate today = LocalDate.now();
// log.debug("loadAllKeys from {} on {} {} today", getTableName(), dateField, canBeGreat ? ">=" : "=");
// List<Long> keys;
// try {
// keys = jdbcTemplate.query("select id from " + getTableName() + " where " + dateField + (canBeGreat ? " >= ?" : " = ?"),
// (resultSet, i) -> resultSet.getLong("id"),
// new Object[]{today});
// } catch (Throwable e) {
// log.error("At load keys from {} (by field {}): {}", getTableName(), dateField, ExceptionUtils.getStackTrace(e));
// throw e;
// }
}
@Override
public Collection<SearchProxyMoneyBalanceRegister> load(Collection<Long> keys) {
Map<String, Collection<Long>> paramMap = Collections.singletonMap("ids", keys);

111
clearing-parent/imdg-hist/src/main/java/ru/spcex/clearing/historyimdg/services/HazelcastLifecycleSupport.java Normal file
View file

@ -0,0 +1,111 @@
package ru.spcex.clearing.historyimdg.services;
import com.hazelcast.core.HazelcastInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import ru.spcex.platform.imdg.iml.hazelcast.util.HazelcastHelper;
import java.util.Collection;
@Service
public class HazelcastLifecycleSupport implements InitializingBean, DisposableBean {
private final Logger log = LoggerFactory.getLogger(getClass());
private final HazelcastInstance hazelcastServerInstance;
private final Collection<String> historyMapNames;
@Autowired
public HazelcastLifecycleSupport(@Qualifier("hazelcastInstanceImdg") HazelcastInstance hazelcastServerInstance, Collection<String> historyMapNames) {
this.hazelcastServerInstance = hazelcastServerInstance;
this.historyMapNames = historyMapNames;
}
@Override
public void afterPropertiesSet() throws Exception {
long loadTime = System.currentTimeMillis();
log.debug("init maps started");
for (String mapName : hazelcastServerInstance.getConfig().getMapConfigs().keySet()) {
//зачем historyMapNames - изначально чтобы соотв. MapStore не делал loadAllKeys
//по факту в Histry мапсторах метод loadAllKeys должен быть всегда переопределен в emptyList()
//можно потом убрать
if (historyMapNames.contains(mapName)) {
continue;
}
log.debug("configured '{}' map for hazelcast search server. size: {}", mapName, hazelcastServerInstance.getMap(mapName).size());
}
loadTime = System.currentTimeMillis() - loadTime;
log.info("All map load time {} ms", loadTime);
HazelcastHelper.imdgSystem_setStorageState(true, hazelcastServerInstance);
}
@Override
public void destroy() throws Exception {
}
//package com.moex.corp.search.service;
//
//import com.hazelcast.core.HazelcastInstance;
//import com.moex.platform.errors.TextErrorService;
//import com.moex.platform.hazelcast.HazelcastCommon;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.DisposableBean;
//import org.springframework.beans.factory.InitializingBean;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.annotation.Qualifier;
//import org.springframework.stereotype.Service;
//
//import java.util.Collection;
//
//@SuppressWarnings("Duplicates")
//@Service
//public class HazelcastLifecycleSupport implements InitializingBean, DisposableBean {
// /**
// * Рабочая версия БД. Требуется вручную сверять с DDL.sql и накручивать эту переменную.
// */
// public static final String CHECK_DB_VERSION = "2.39";
//
// private final Logger log = LoggerFactory.getLogger(this.getClass());
//
// private final HazelcastInstance hazelcastServerInstance;
// private final Collection<String> historyMapNames;
//
// @Autowired
// public HazelcastLifecycleSupport(HazelcastInstance hazelcastServerInstance,
// @Qualifier("historyMapNames") Collection<String> historyMapNames) {
// this.hazelcastServerInstance = hazelcastServerInstance;
// this.historyMapNames = historyMapNames;
// }
//
// @Override
// public void afterPropertiesSet() {
// long loadTime = System.currentTimeMillis();
// log.debug("init maps started");
// for (String mapName : hazelcastServerInstance.getConfig().getMapConfigs().keySet()) {
// if (historyMapNames.contains(mapName)) {
// continue;
// }
// log.debug("configured '{}' map for hazelcast search server. size: {}", mapName, hazelcastServerInstance.getMap(mapName).size());
// }
// loadTime = System.currentTimeMillis() - loadTime;
// log.info("All map load time {} ms", loadTime);
// HazelcastCommon.otcSystem_setStorageState(true, hazelcastServerInstance);
//
// TextErrorService.setHazelcast(hazelcastServerInstance);
//
// }
//
// @Override
// public void destroy() {
// hazelcastServerInstance.shutdown();
// }
//
// public HazelcastInstance getHazelcastServerInstance() {
// return hazelcastServerInstance;
// }
//}
}

6
clearing-parent/imdg-hist/src/main/resources/application.properties
View file

@ -1,6 +1,6 @@
imdg.hist.hazelcast.listenPort=5701
imdg.hist.hazelcast.login=dev
imdg.hist.hazelcast.password=dev-pass
imdg.hist.hazelcast.listenPort=5702
imdg.hist.hazelcast.login=dev-hist
imdg.hist.hazelcast.password=dev-pass-hist
imdg.hist.hazelcast.cluster-members[0]=127.0.0.1
imdg.hist.database.login=clearing
imdg.hist.database.password=Aa111111

6
platform-parent/platform-imdg-api-hazelcast-impl/src/main/java/ru/spcex/platform/imdg/iml/hazelcast/adapter/ImdgHazelcast.java
View file

@ -232,6 +232,12 @@ public class ImdgHazelcast<T extends SpcexObjectBase> implements Imdg<T> {
return result;
}
@Override
public Collection<Long> getCollectionIdsByPredicate(ImdgPredicate prdct) {
Predicate hazelcastPredicate = ((ImdgPredicateHazelcast) prdct).getRawPredicate();
return new ArrayList<>(map.keySet(hazelcastPredicate));
}
@Override
public Long nextIDSequenceFor() {
return idGenerator.newId();

4
platform-parent/platform-imdg-api/src/main/java/ru/spcex/platform/imdg/api/Imdg.java
View file

@ -64,6 +64,10 @@ public interface Imdg<T extends SpcexObjectBase> {
throw new UnsupportedOperationException("not implemented getCollectionIdsBySQL");
}
default Collection<Long> getCollectionIdsByPredicate(ImdgPredicate prdct) {
throw new UnsupportedOperationException("not implemented getCollectionIdsByPredicate");
}
default Collection<T> getAllValues() {
throw new UnsupportedOperationException("not implemented getAllValues");
}

2
z-distr/pom.xml
View file

@ -147,7 +147,7 @@
<configuration>
<fileSets>
<fileSet>
<sourceFile>${folder_root_clearing_imdg}/target/imdg.jar</sourceFile>
<sourceFile>${folder_root_clearing_imdg}/target/imdg-exec.jar</sourceFile>
<destinationFile>${folder.clearing.distr.bin}/imdg.jar</destinationFile>
</fileSet>
<fileSet>