sessions 5 step: batch put all optimization

This commit is contained in:
ialbert 2024-07-23 14:37:39 +03:00
parent 05893af032
commit 40ec9fde5f
6 changed files with 201 additions and 108 deletions

View file

@ -0,0 +1,50 @@
package ru.spcex.clearing.component.predicate.cash.registry;
import ru.clearing.classes.statics.data.registry.Registry;
import ru.spcex.clearing.service.cash.impl.RegistryCashA__F;
import ru.spcex.platform.enumeration.RegistryInstrumentType;
import ru.spcex.platform.enumeration.RegistryTradingParams;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicate;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicateBuilder;
import ru.spcex.platform.imdg.api.predicate.specific.RegistryCodeSqlBuilder;
import ru.spcex.platform.utils.enumeration.IEnumKey;
public class AssetByObligationCashedPredicate {
private final RegistryCashA__F.CustomKey key;
private final RegistryTradingParams params;
public static AssetByObligationCashedPredicate getPredicate(Registry rgs) {
return new AssetByObligationCashedPredicate(rgs);
}
public ImdgPredicate cashed(ImdgPredicateBuilder pb, RegistryCashA__F cash) {
ImdgPredicate prdct = pb.and(
pb.sql(RegistryCodeSqlBuilder.getInstance(params).build()),
pb.equals("tradingClearingRegistryId", key.tcrId()),
pb.equals("securitySymbol", key.secSymbol()),
pb.equals("companyId", key.cmpId())
);
if (cash != null) {
return pb.cashed(prdct, cash, key);
} else {
return prdct;
}
}
private AssetByObligationCashedPredicate(Registry rgs) {
if (rgs == null) throw new IllegalStateException("cannot obtain cash for null obligation");
RegistryTradingParams params;
if (IEnumKey.getEnumByKey(RegistryInstrumentType.class, rgs.getRegistryInstrumentType()) == RegistryInstrumentType.S) {
params = RegistryTradingParams.AS_F;
} else if (IEnumKey.getEnumByKey(RegistryInstrumentType.class, rgs.getRegistryInstrumentType()) == RegistryInstrumentType.M) {
params = RegistryTradingParams.AM_F;
} else {
throw new IllegalStateException("unknown RegistryInstrumentType '%s' for rgs.id=%d".formatted(
rgs.getRegistryInstrumentType(),
rgs.getId()
));
}
this.params = params;
this.key = RegistryCashA__F.key(rgs);
}
}

View file

@ -0,0 +1,37 @@
package ru.spcex.clearing.component.predicate.cash.registry;
import ru.clearing.classes.statics.data.registry.Registry;
import ru.spcex.clearing.service.cash.impl.RegistryCashAssetCash;
import ru.spcex.platform.enumeration.RegistryTradingParams;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicate;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicateBuilder;
import ru.spcex.platform.imdg.api.predicate.specific.RegistryCodeSqlBuilder;
public class AssetByTcrCompanyAccountCashedPredicate {
private final RegistryCashAssetCash.CustomKey key;
private final RegistryTradingParams params;
public static AssetByTcrCompanyAccountCashedPredicate getPredicate(Registry rgs, RegistryTradingParams params) {
return new AssetByTcrCompanyAccountCashedPredicate(rgs, params);
}
public ImdgPredicate cashed(ImdgPredicateBuilder pb, RegistryCashAssetCash cash) {
ImdgPredicate prdct = pb.and(
pb.sql(RegistryCodeSqlBuilder.getInstance(params).build()),
pb.equals("tradingClearingRegistryId", key.tcrId()),
pb.equals("companyId", key.cmpId()),
pb.equals("accountId", key.accId()),
pb.equals("securityId", key.secId())
);
if (cash != null) {
return pb.cashed(prdct, cash, key);
} else {
return prdct;
}
}
private AssetByTcrCompanyAccountCashedPredicate(Registry rgs, RegistryTradingParams params) {
this.params = params;
this.key = RegistryCashAssetCash.key(rgs, params);
}
}

View file

@ -15,7 +15,7 @@ public class RegistryCashA__F extends CashV2<RegistryCashA__F.CustomKey, Registr
super(name);
}
private RegistryCashA__F.CustomKey key(Registry obligation) {
public static RegistryCashA__F.CustomKey key(Registry obligation) {
if (obligation == null) throw new IllegalStateException("cannot obtain cash for null obligation");
RegistryTradingParams rgsCde = null;
@ -50,18 +50,7 @@ public class RegistryCashA__F extends CashV2<RegistryCashA__F.CustomKey, Registr
return new CustomKey(RegistryUtil.rgsCdeWithoutCapacity(rgs), rgs.getTradingClearingRegistryId(), rgs.getSecuritySymbol(), rgs.getCompanyId());
}
public static class CustomKey {
private final String rgsCde;
private final Long tcrId;
private final String secSymbol;
private final Long cmpId;
private CustomKey(String rgsCde, Long tcrId, String secSymbol, Long cmpId) {
this.rgsCde = rgsCde;
this.tcrId = tcrId;
this.secSymbol = secSymbol;
this.cmpId = cmpId;
}
public static record CustomKey(String rgsCde, Long tcrId, String secSymbol, Long cmpId) {
@Override
public boolean equals(Object o) {

View file

@ -15,7 +15,7 @@ public class RegistryCashAssetCash extends CashV2<RegistryCashAssetCash.CustomKe
super(name);
}
private CustomKey key(Registry rgs, RegistryTradingParams params) {
public static CustomKey key(Registry rgs, RegistryTradingParams params) {
return new CustomKey(
RegistryUtil.clearingCode(params.cpcty((RegistryCapacity) null)),
rgs.getTradingClearingRegistryId(),
@ -47,20 +47,24 @@ public class RegistryCashAssetCash extends CashV2<RegistryCashAssetCash.CustomKe
return new CustomKey(RegistryUtil.rgsCdeWithoutCapacity(rgs), rgs.getTradingClearingRegistryId(), rgs.getSecurityId(), rgs.getCompanyId(), rgs.getAccountId());
}
public static class CustomKey {
private final String rgsCde;
private final Long tcrId;
private final Long secId;
private final Long cmpId;
private final Long accId;
public CustomKey(String rgsCde, Long tcrId, Long secId, Long cmpId, Long accId) {
this.rgsCde = rgsCde;
this.tcrId = tcrId;
this.secId = secId;
this.cmpId = cmpId;
this.accId = accId;
}
public static record CustomKey(String rgsCde,
Long tcrId,
Long secId,
Long cmpId,
Long accId) {
// private final String rgsCde;
// private final Long tcrId;
// private final Long secId;
// private final Long cmpId;
// private final Long accId;
//
// public CustomKey(String rgsCde, Long tcrId, Long secId, Long cmpId, Long accId) {
// this.rgsCde = rgsCde;
// this.tcrId = tcrId;
// this.secId = secId;
// this.cmpId = cmpId;
// this.accId = accId;
// }
@Override
public boolean equals(Object o) {

View file

@ -6,6 +6,7 @@ import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -18,6 +19,8 @@ import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import ru.clearing.classes.statics.data.registry.Registry;
import ru.spcex.clearing.component.predicate.cash.registry.AssetByObligationCashedPredicate;
import ru.spcex.clearing.component.predicate.cash.registry.AssetByTcrCompanyAccountCashedPredicate;
import ru.spcex.clearing.error.ClearingError;
import ru.spcex.clearing.error.RgsError;
import ru.spcex.clearing.imdg.IMDGDistributedNames;
@ -44,7 +47,9 @@ import ru.spcex.platform.enumeration.RegistryUnit;
import ru.spcex.platform.enumeration.Section;
import ru.spcex.platform.enumeration.SessionType;
import ru.spcex.platform.imdg.api.Imdg;
import ru.spcex.platform.imdg.api.ImdgId;
import ru.spcex.platform.imdg.api.ImdgProvider;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicate;
import ru.spcex.platform.imdg.api.predicate.ImdgPredicateBuilder;
import ru.spcex.platform.imdg.api.predicate.specific.RegistryCodeSqlBuilder;
import ru.spcex.platform.utils.collection.Pair;
@ -60,6 +65,7 @@ import static ru.spcex.platform.utils.number.BigDecimalUtil.safeBD;
public class InspectionObligationsV2 implements ISessionStage {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Imdg<Registry> registryImdg;
private final ImdgId idGen;
private final RegistryManager registryManager;
private final IMessageResolver msgResolver = new SimpleMessageResolver();
private SessionType sessionType;
@ -75,7 +81,8 @@ public class InspectionObligationsV2 implements ISessionStage {
RegistryManager registryManager,
AssetTBFProcessing assets,
GatewayRequester gateway, PlanBalanceCalc planBalanceCalc) {
this.registryImdg = imdgProvider.getImdg(IMDGDistributedNames.Map_Registry, Registry.class);
this.registryImdg = imdgProvider.getCashingImdg(IMDGDistributedNames.Map_Registry, Registry.class, null);
this.idGen = imdgProvider.getImdgIdGenerator();
this.registryManager = registryManager;
this.assets = assets;
this.gateway = gateway;
@ -121,6 +128,7 @@ public class InspectionObligationsV2 implements ISessionStage {
RegistryStatus.POOL.getKey());
Collection<Registry> registriesToProcess = registryImdg.getCollectionObjectsBySQL(sqlCondition);
Map<Long, Registry> rgssToStore = new HashMap<>(registriesToProcess.size()); //вторые ноги у сделок - нужна ли оптимизация?
List<Map.Entry<Long, List<Registry>>> registriesByGroupSorted = registriesToProcess.stream()
.collect(Collectors.groupingBy(Registry::getGroupId))
.entrySet()
@ -140,8 +148,8 @@ public class InspectionObligationsV2 implements ISessionStage {
})
.toList();
List<Registry> assetsToUpdate = new ArrayList<>();
log.info("found {} ({} groups) registries by sql: {}", registriesToProcess.size(), registriesByGroupSorted.size(), sqlCondition);
ImdgPredicateBuilder rgsPb = registryImdg.predicateBuilder();
GROUP:
for (Map.Entry<Long, List<Registry>> entry : registriesByGroupSorted) {
List<Registry> group = entry.getValue();
@ -159,9 +167,10 @@ public class InspectionObligationsV2 implements ISessionStage {
checkResults.add(new CheckResult(obligation, false));
continue;
}
String sqlAssetRegistryCondition = searchAssetsByObligationSql(obligation);
log.debug("search asset by registry.id={} sql {}", obligation.getId(), sqlAssetRegistryCondition);
Registry asset = a__fCash.getOrFind(obligation, () -> registryImdg.getFirstObjectBySQL(sqlAssetRegistryCondition));
ImdgPredicate assetPrdct = AssetByObligationCashedPredicate
.getPredicate(obligation)
.cashed(rgsPb, a__fCash);
Registry asset = registryImdg.getFirstObjectByPredicate(assetPrdct);
if (asset == null) {
log.warn("Not found asset by registry.id={}", obligation.getId());
isUncovered = true;
@ -179,7 +188,7 @@ public class InspectionObligationsV2 implements ISessionStage {
Runnable failGroup = () -> Stream.concat(group.stream(), getRefundDateRgsIfPresent(group).stream())
.forEach(registry -> {
registry.setComment(msgResolver.resolve(RgsError.A__tNotFound));
updateRegistryStatus(registry, RegistryStatus.FAIL);
updateRegistryStatus(registry, RegistryStatus.FAIL, rgssToStore);
});
RegistryInstrumentType mOrS = IEnumKey.getEnumByKey(RegistryInstrumentType.class, rgs.getRegistryInstrumentType());
if (mOrS == null) {
@ -191,9 +200,11 @@ public class InspectionObligationsV2 implements ISessionStage {
if (Section.MKR.equalsByKey(rgs.getSection()) && mOrS.equals(RegistryInstrumentType.S)) {
continue;
}
Optional<Registry> a__t = a___Cash.getOrFindO(rgs, A__T.type(mOrS), () ->
assets.searchByTcrCompanyAccount(rgs, A__T.type(mOrS)));
if (a__t.isEmpty()) {
ImdgPredicate assetPredicate = AssetByTcrCompanyAccountCashedPredicate
.getPredicate(rgs, A__T.type(mOrS))
.cashed(rgsPb, a___Cash);
Registry a__t = registryImdg.getFirstObjectByPredicate(assetPredicate);
if (a__t == null) {
log.debug("groupId {}, {}.id={} - A**T not found. settings FAIL to group",
entry.getKey(), rgs.getRegistryCode(), rgs.getId());
failGroup.run();
@ -215,7 +226,7 @@ public class InspectionObligationsV2 implements ISessionStage {
.ifPresent(chk -> chk.isUncovered = true);
}
}
defineStatusAndUpdateRegistry(checkResults, group);
defineStatusAndUpdateRegistry(checkResults, group, rgssToStore);
if (checkResults.stream().noneMatch(checkResult -> checkResult.isUncovered)) {
Instant now = Instant.now();
//по каждому регистру OM*T, TM*T, OS*T, TS*T из одной группы
@ -223,86 +234,70 @@ public class InspectionObligationsV2 implements ISessionStage {
{
RegistryInstrumentType mOrS = IEnumKey.getEnumByKey(RegistryInstrumentType.class, registry.getRegistryInstrumentType());
if (mOrS == null) continue;
Optional<Registry> a__t = a___Cash.getOrFindO(registry, A__T.type(mOrS), ()
-> assets.searchByTcrCompanyAccount(registry, A__T.type(mOrS)));
Optional<Registry> a__b = a__t.flatMap(a__tFound -> a___Cash.getOrFind(
a__tFound,
A__B.type(mOrS),
() -> assets
.searchByTcrCompanyAccount(a__tFound, A__B.type(mOrS)).orElseGet(() -> copyB(a__tFound))));
// Optional<Registry> a__b = a__t.map(a__tFound -> assets
// .searchByTcrCompanyAccount(a__tFound, A__B.type(mOrS))
// .orElseGet(() -> copyB(a__tFound)));
Optional<Registry> a__f = a__t.flatMap(a__tFound -> a___Cash.getOrFind(
a__tFound,
A__F.type(mOrS),
() -> assets
.searchByTcrCompanyAccount(a__tFound, A__F.type(mOrS)).orElseGet(() -> copyF(a__tFound))));
// Optional<Registry> a__f = a__t.map(a__tFound -> assets
// .searchByTcrCompanyAccount(a__tFound, A__F.type(mOrS))
// .orElseGet(() -> copyF(a__tFound)));
log.debug("registry {}.id={}: {}", registry.getRegistryCode(), registry.getId(),
a__t.map(r -> "A**T.id=%d/A**B.id=%d/A**F.id=%d"
.formatted(a__t.get().getId(), a__b.get().getId(), a__f.get().getId()))
.orElse("A**T/A**B/A**F not found"));
if (a__t.isPresent()) {
ImdgPredicate a__tPrdct = AssetByTcrCompanyAccountCashedPredicate
.getPredicate(registry, A__T.type(mOrS))
.cashed(rgsPb, a___Cash);
Registry a__t = registryImdg.getFirstObjectByPredicate(a__tPrdct);
Registry a__b = null;
Registry a__f = null;
if (a__t != null) {
ImdgPredicate a__bPrdct = AssetByTcrCompanyAccountCashedPredicate
.getPredicate(registry, A__B.type(mOrS))
.cashed(rgsPb, a___Cash);
ImdgPredicate a__fPrdct = AssetByTcrCompanyAccountCashedPredicate
.getPredicate(registry, A__F.type(mOrS))
.cashed(rgsPb, a___Cash);
a__b = Optional.ofNullable(
registryImdg.getFirstObjectByPredicate(a__bPrdct)
).orElseGet(() -> copyB(a__t));
a__f = Optional.ofNullable(
registryImdg.getFirstObjectByPredicate(a__fPrdct)
).orElseGet(() -> copyF(a__t));
}
{
String msg;
if (a__t != null) {
msg = "A**T.id=%d/A**B.id=%d/A**F.id=%d".formatted(
a__t.getId(), a__b.getId(), a__f.getId()
);
} else {
msg = "A**T/A**B/A**F not found";
}
log.debug("registry {}.id={}: {}", registry.getRegistryCode(), registry.getId(), msg);
}
if (a__t != null) {
BigDecimal amount = safeBD(registry.getBalance());
if (IEnumKey.getEnumByKey(RegistryDesignation.class, registry.getRegistryDesignation()) == RegistryDesignation.O) {
amount = amount.negate();
} else if (IEnumKey.getEnumByKey(RegistryDesignation.class, registry.getRegistryDesignation()) == RegistryDesignation.T) {
amount = amount;
}
a__b.get().setSessionId(sessionId);
assetsCashing.process(a__b.get(), a__t.get(), a__f.get(), amount);
assetsToUpdate.add(a__b.get());
assetsToUpdate.add(a__f.get());
a__b.setSessionId(sessionId);
assetsCashing.process(a__b, a__t, a__f, amount);
rgssToStore.put(a__b.getId(), a__b);
rgssToStore.put(a__f.getId(), a__f);
}
}
}
}
}
for (Registry registry : assetsToUpdate) {
if (registry.getId() != null) {
registryImdg.update(registry);
} else {
registryImdg.insert(registry);
}
}
registryImdg.putAll(rgssToStore);
planBalanceCalc.stageRevision2(sessionId);
return new StageResult(null, true);
}
private String searchAssetsByObligationSql(Registry obligation) {
RegistryTradingParams counterRegistryTradingParams = null;
if (IEnumKey.getEnumByKey(RegistryInstrumentType.class, obligation.getRegistryInstrumentType()) == RegistryInstrumentType.S) {
counterRegistryTradingParams = new RegistryTradingParams(RegistryDesignation.A,
RegistryInstrumentType.S,
null,
RegistryUnit.F);
} else if (IEnumKey.getEnumByKey(RegistryInstrumentType.class, obligation.getRegistryInstrumentType()) == RegistryInstrumentType.M) {
counterRegistryTradingParams = new RegistryTradingParams(RegistryDesignation.A,
RegistryInstrumentType.M,
null,
RegistryUnit.F);
}
return String.format("%s and " +
"tradingClearingRegistryId = '%s' and " +
"securitySymbol = '%s' and " +
"companyId = %d",
RegistryCodeSqlBuilder.getInstance(counterRegistryTradingParams).build(),
obligation.getTradingClearingRegistryId(),
obligation.getSecuritySymbol(),
obligation.getCompanyId());
}
private Registry copyF(Registry rgs) {
Registry rgsF = rgs.clone();
RegistryManager.zeroState(rgsF);
rgsF.setRegistryUnit(RegistryUnit.F.getKey());
rgsF.setRegistryCode(RegistryUtil.clearingCode(rgsF));
rgsF.setId(idGen.nextId());
registryImdg.insert(rgsF);
log.debug("created {}.id={} by {}.id={}", rgsF.getRegistryCode(), rgsF.getId(), rgs.getRegistryCode(), rgs.getId());
return rgsF;
@ -313,12 +308,13 @@ public class InspectionObligationsV2 implements ISessionStage {
RegistryManager.zeroState(rgsB);
rgsB.setRegistryUnit(RegistryUnit.B.getKey());
rgsB.setRegistryCode(RegistryUtil.clearingCode(rgsB));
rgsB.setId(idGen.nextId());
registryImdg.insert(rgsB);
log.debug("created {}.id={} by {}.id={}", rgsB.getRegistryCode(), rgsB.getId(), rgs.getRegistryCode(), rgs.getId());
return rgsB;
}
private void defineStatusAndUpdateRegistry(List<CheckResult> checkResults, List<Registry> registries) {
private void defineStatusAndUpdateRegistry(List<CheckResult> checkResults, List<Registry> registries, Map<Long, Registry> rgssToStore) {
boolean isOneUncovered = checkResults.stream().anyMatch(checkResult -> checkResult.isUncovered);
log.debug("groupId {}, uncovered: {}", registries.stream().findFirst().map(Registry::getGroupId).orElse(null), isOneUncovered);
String commentErr = msgResolver.resolve(ClearingError.InsecurityObligation, checkResults.stream()
@ -334,17 +330,17 @@ public class InspectionObligationsV2 implements ISessionStage {
).findFirst();
checkResult.registry.setComment(commentErr);
if (checkResult.isUncovered) {
updateRegistryStatus(checkResult.registry, uncvStatus());
tRegistryWithSameCompany.ifPresent(registry -> updateRegistryStatus(registry, failStatus()));
updateRegistryStatus(checkResult.registry, uncvStatus(), rgssToStore);
tRegistryWithSameCompany.ifPresent(registry -> updateRegistryStatus(registry, failStatus(), rgssToStore));
} else {
updateRegistryStatus(checkResult.registry, failStatus());
tRegistryWithSameCompany.ifPresent(registry -> updateRegistryStatus(registry, failStatus()));
updateRegistryStatus(checkResult.registry, failStatus(), rgssToStore);
tRegistryWithSameCompany.ifPresent(registry -> updateRegistryStatus(registry, failStatus(), rgssToStore));
}
}
//проверяем есть ли второй день для сделки (он не входит в пул, поэтому ищем отдельно)
getRefundDateRgsIfPresent(registries).forEach(rgs -> updateRegistryStatus(rgs, RegistryStatus.FAIL));
getRefundDateRgsIfPresent(registries).forEach(rgs -> updateRegistryStatus(rgs, RegistryStatus.FAIL, rgssToStore));
} else {
registries.forEach(registry -> updateRegistryStatus(registry, RegistryStatus.OK));
registries.forEach(registry -> updateRegistryStatus(registry, RegistryStatus.OK, rgssToStore));
}
}
@ -375,6 +371,14 @@ public class InspectionObligationsV2 implements ISessionStage {
registryImdg.update(registry);
}
private void updateRegistryStatus(Registry registry, RegistryStatus registryStatus, Map<Long, Registry> rgss) {
log.debug("Update registry.id: {} to {}", registry.getId(), registryStatus.getKey());
registry.setRegistryStatus(registryStatus.getKey());
registry.setUpdated(Instant.now());
rgss.put(registry.getId(), registry);
//registryImdg.update(registry);
}
private static final class CheckResult {
private final Registry registry;
private boolean isUncovered;

View file

@ -13,10 +13,11 @@ public abstract class CashV2<K, T> {
private long missRate = 0;
private final Map<K, T> cash;
private final String name;
protected int threshold = 200_000;
public CashV2() {
this.cash = new HashMap<>();
this.name = null;
this.name = "[unknown]";
}
public CashV2(String name) {
@ -26,7 +27,7 @@ public abstract class CashV2<K, T> {
public void clear() {
log.info("cash {}: size {}, hit rate {}, miss rate {}",
name != null ? name : "[unknown]",
name,
cash.size(),
hitRate,
missRate);
@ -38,7 +39,7 @@ public abstract class CashV2<K, T> {
public void store(T obj) {
if (obj == null || !needToStore(obj)) return;
K key = extractKey(obj);
cash.put(key, obj);
putIfThresholdNotReached(key, obj);
}
protected Optional<T> get(K key) {
@ -53,7 +54,7 @@ public abstract class CashV2<K, T> {
if (by.isEmpty()) {
T t = supplier.get();
if (t != null && needToStore(t)) {
cash.put(key, t);
putIfThresholdNotReached(key, t);
by = Optional.of(t);
}
}
@ -66,7 +67,7 @@ public abstract class CashV2<K, T> {
Optional<T> foundAnew = supplier.get();
if (foundAnew.isPresent()) {
if (needToStore(foundAnew.get())) {
cash.put(key, foundAnew.get());
putIfThresholdNotReached(key, foundAnew.get());
}
fromCash = foundAnew;
}
@ -74,6 +75,14 @@ public abstract class CashV2<K, T> {
return fromCash;
}
private void putIfThresholdNotReached(K key, T value) {
if (cash.size() < threshold) {
cash.put(key, value);
} else {
log.warn("cash {}: size {} threshold reached. will not put new value in cash", name, value);
}
}
protected abstract K extractKey(T obj);
protected boolean needToStore(T obj) {