treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

http://jira.mfd.msk:8088/browse/CLS-754

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2024-09-12 18:15:10 +03:00
parent 3fd5ca37ab
commit bcd1aa2b40
4 changed files with 246 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

149
clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/base/SimpleObjectMapStore.java
View file

@ -1,6 +1,23 @@
package ru.spcex.clearing.imdg.base;
import com.hazelcast.core.MapStore;
import java.math.BigDecimal;
import java.sql.ParameterMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.sql.Types;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -8,18 +25,11 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.SqlTypeValue;
import org.springframework.jdbc.core.StatementCreatorUtils;
import org.springframework.lang.NonNull;
import ru.spcex.clearing.imdg.util.ColumnMetaData;
import ru.spcex.clearing.imdg.util.PostgresColumnTypeUtil;
import ru.spcex.platform.classes.base.SpcexObjectBase;
import ru.spcex.platform.utils.text.TextUtil;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDate;
import java.util.*;
import java.util.stream.Collectors;
/**
* MapStore для выгрузки обычных объектов
*
@ -30,6 +40,7 @@ public abstract class SimpleObjectMapStore<T extends SpcexObjectBase> implements
protected final Logger log = LoggerFactory.getLogger(this.getClass());
protected final JdbcTemplate jdbcTemplate;
protected SimpleDateFormat DB_DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd");
protected Map<String, ColumnMetaData> columnMetaDataMap;
protected SimpleObjectMapStore(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
@ -149,10 +160,12 @@ public abstract class SimpleObjectMapStore<T extends SpcexObjectBase> implements
}
protected void batchInsertUpdate(@NonNull String insertStatement, @NonNull List<Object[]> args) {
updateMetaData();
try {
int[][] ret = jdbcTemplate.batchUpdate(insertStatement, args, BATCH_SIZE, (ps, params) -> {
for (int i = 0; i < params.length; i++) {
Object value = params[i];
value = normalizeValue(value, ps.getParameterMetaData(), columnMetaDataMap, i, insertStatement, params);
try {
StatementCreatorUtils.setParameterValue(ps, i + 1, SqlTypeValue.TYPE_UNKNOWN, value);
} catch (SQLException e) {
@ -168,6 +181,112 @@ public abstract class SimpleObjectMapStore<T extends SpcexObjectBase> implements
}
}
Object normalizeValue(Object value, ParameterMetaData metaData, Map<String, ColumnMetaData> columnMetaDataMap, int i, String insertStatement, Object[] params) throws SQLException {
if (value == null)
return null;
i++; // нумерация в ParameterMetaData с 1
final int type = metaData.getParameterType(i);
final String fieldName = getFields()[i - 1].toLowerCase();
if (columnMetaDataMap == null) {
log.trace("Column metadata was empty, can not verify {}.{}", getTableName(), fieldName);
}
if ((type == Types.VARCHAR || type == Types.CHAR) && value instanceof String stringValue) {
int length;
if (columnMetaDataMap == null) {
length = metaData.getPrecision(i + 1);
} else {
// в postgresql для String ParameterMetaData.getPrecision возвращает 0, по этому используется другой способ через columnMetaDataMap.
length = columnMetaDataMap.get(fieldName).getPrecision();
}
if (length > 0 && stringValue.length() > length) {
value = stringValue.substring(0, length);
log.error("String value for insert is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]",
insertStatement, Arrays.toString(params), i, stringValue, value);
}
} else if ((type == Types.DECIMAL || type == Types.NUMERIC || type == Types.BIGINT || type == Types.INTEGER) &&
(value instanceof BigDecimal || value instanceof Long || value instanceof Integer)) {
if (value instanceof Integer && (type == Types.INTEGER || type == Types.BIGINT)) {
return value; // value ok
}
if (value instanceof Long vl) {
if (type == Types.BIGINT) {
return value; // value ok
}
if (type == Types.INTEGER) {
if (vl > Integer.MAX_VALUE) {
value = Integer.MAX_VALUE;
log.error("Long value for insert is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]",
insertStatement, Arrays.toString(params), i, vl, value);
}
if (vl < Integer.MIN_VALUE) {
value = Integer.MIN_VALUE;
log.error("Long value for insert is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]",
insertStatement, Arrays.toString(params), i, vl, value);
}
return value;
}
}
BigDecimal bigDecimalValue;
if (value instanceof Integer intValue) {
bigDecimalValue = BigDecimal.valueOf(intValue);
} else if (value instanceof Long longValue) {
bigDecimalValue = BigDecimal.valueOf(longValue);
} else {
bigDecimalValue = (BigDecimal) value;
}
if (type == Types.BIGINT) {
if (bigDecimalValue.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) {
Object oldValue = value;
value = Long.MAX_VALUE;
log.error("Double value for insert as BIGINT is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]",
insertStatement, Arrays.toString(params), i, oldValue, value);
} else if (bigDecimalValue.compareTo(BigDecimal.valueOf(Long.MIN_VALUE)) < 0) {
Object oldValue = value;
value = Long.MIN_VALUE;
log.error("Double value for insert as BIGINT is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]",
insertStatement, Arrays.toString(params), i, oldValue, value);
}
return value; // value ok
}
if (type == Types.INTEGER) {
if (bigDecimalValue.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) > 0) {
Object oldValue = value;
value = Integer.MAX_VALUE;
log.error("Double value for insert as INTEGER is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]",
insertStatement, Arrays.toString(params), i, oldValue, value);
} else if (bigDecimalValue.compareTo(BigDecimal.valueOf(Integer.MIN_VALUE)) < 0) {
Object oldValue = value;
value = Integer.MIN_VALUE;
log.error("Double value for insert as INTEGER is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]",
insertStatement, Arrays.toString(params), i, oldValue, value);
}
return value; // value ok
}
int dbScalePartLength;
int precision;
if (columnMetaDataMap == null) {
dbScalePartLength = metaData.getScale(i);
precision = metaData.getPrecision(i);
} else {
dbScalePartLength = columnMetaDataMap.get(fieldName).getScale();
precision = columnMetaDataMap.get(fieldName).getPrecision();
}
if (!checkBigDecimalSize(bigDecimalValue, precision, dbScalePartLength)) {
boolean negative = bigDecimalValue.signum() < 0;
value = new BigDecimal(BigDecimal.valueOf(Math.pow(10, precision - dbScalePartLength)).longValue() - 1);
if (negative)
value = ((BigDecimal) value).negate();
log.error("BigDecimal value for insert is too large [ insertSql: {}, params: {}, i:{}, incorrect value : {}, normalized value: {} ]",
insertStatement, Arrays.toString(params), i, bigDecimalValue, value);
}
}
return value;
}
protected Instant getInstantFromTimestamp(ResultSet rs, String column) throws SQLException {
Timestamp date = rs.getTimestamp(column);
return date != null ? date.toInstant() : null;
@ -191,4 +310,16 @@ public abstract class SimpleObjectMapStore<T extends SpcexObjectBase> implements
return TextUtil.format(INSERT_TEMPLATE, params);
}
private void updateMetaData() {
if (columnMetaDataMap == null) synchronized (this) {
columnMetaDataMap = PostgresColumnTypeUtil.extractTableMetadata(jdbcTemplate, getTableName());
}
}
private boolean checkBigDecimalSize(BigDecimal value, int dbSize, int dbScale) {
if (value == null)
return true;
return value.precision() - value.scale() <= dbSize - dbScale;
}
}

43
clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java
View file

@ -4,6 +4,14 @@ import com.hazelcast.config.MapStoreConfig;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IMap;
import com.hazelcast.core.IdGenerator;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
@ -16,10 +24,6 @@ import ru.spcex.clearing.imdg.base.SimpleObjectMapStore;
import ru.spcex.clearing.imdg.object.DbVersionMapStore;
import ru.spcex.platform.imdg.iml.hazelcast.util.HazelcastHelper;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
//todo почистить класс
public abstract class AbstractHazelcastLifecycleSupport implements InitializingBean, DisposableBean {
private final Logger log = LoggerFactory.getLogger(this.getClass());
@ -48,7 +52,7 @@ public abstract class AbstractHazelcastLifecycleSupport implements InitializingB
@Override
public void afterPropertiesSet() {
// HazelcastCommon.loadClassesVersionsByStorage(hazelcastServerInstance, getSerialVersionUIDClasses());
// databaseVersionCheck();
databaseVersionCheck();
long loadTime = System.currentTimeMillis();
@ -119,11 +123,38 @@ public abstract class AbstractHazelcastLifecycleSupport implements InitializingB
HazelcastHelper.imdgSystem_setStorageState(true, hazelcastServerInstance);
loadTime = System.currentTimeMillis() - loadTime;
log.info("All map load time {} ms", loadTime);
}
@Override
public void destroy() {
hazelcastServerInstance.shutdown();
}
private void databaseVersionCheck() {
log.debug("Validating database version, expected \"{}\"", getCheckDbVersion());
String dbVersion = jdbcTemplate.queryForObject("SELECT version FROM db_version", String.class);
log.trace("Database version is \"{}\"", dbVersion);
if (dbVersion == null || dbVersion.isBlank()) {
log.error("Database version is empty");
System.exit(1);
} else {
List<Integer> expectedVersion = Arrays.stream(getCheckDbVersion().split("\\."))
.map(Integer::parseInt)
.toList();
List<Integer> actualVersion = Arrays.stream(dbVersion.split("\\."))
.map(Integer::parseInt)
.toList();
if (expectedVersion.size() < 2 || actualVersion.size() < 2) {
log.error("Database version have wrong format, expected = {}, actual = {}", expectedVersion, actualVersion);
System.exit(1);
}
if (!expectedVersion.get(0).equals(actualVersion.get(0)) || expectedVersion.get(1) > actualVersion.get(1)) {
log.error("Database version is older than expected version, expected = {}, actual = {}", expectedVersion, actualVersion);
System.exit(1);
}
}
}
}

40
clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/ColumnMetaData.java Normal file
View file

@ -0,0 +1,40 @@
package ru.spcex.clearing.imdg.util;
public class ColumnMetaData {
private final String name;
private final String dataType;
private final int characterMaximumLength;
private final int numericPrecision;
private final int numericScale;
public ColumnMetaData(String name, String dataType, int characterMaximumLength, int numericPrecision, int numericScale) {
this.name = name;
this.dataType = dataType;
this.characterMaximumLength = characterMaximumLength;
this.numericPrecision = numericPrecision;
this.numericScale = numericScale;
}
public String getName() {
return name;
}
public String getDataType() {
return dataType;
}
public int getPrecision() {
return switch (dataType) {
case "smallint", "integer", "real", "numeric", "bigint", "double precision" -> numericPrecision;
case "character varying" -> characterMaximumLength;
default -> 0;
};
}
public int getScale() {
return switch (dataType) {
case "smallint", "integer", "real", "numeric", "bigint", "double precision" -> numericScale;
default -> 0;
};
}
}

29
clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/PostgresColumnTypeUtil.java Normal file
View file

@ -0,0 +1,29 @@
package ru.spcex.clearing.imdg.util;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
public final class PostgresColumnTypeUtil {
static Logger log = LoggerFactory.getLogger(PostgresColumnTypeUtil.class);
static String SELECT_METADATA_CASE_SENS = "SELECT * FROM information_schema.columns WHERE table_name=?";
static String SELECT_METADATA_CASE_INSENS = "SELECT * FROM information_schema.columns WHERE table_name ilike ?";
public static Map<String, ColumnMetaData> extractTableMetadata(JdbcTemplate template, String tableName) {
List<ColumnMetaData> columnMetaDataList = template.query(tableName.contains("\"") ? SELECT_METADATA_CASE_SENS
: SELECT_METADATA_CASE_INSENS, (rs, rn) -> new ColumnMetaData(
rs.getString("column_name"),
rs.getString("data_type"),
rs.getInt("character_maximum_length"),
rs.getInt("numeric_precision"),
rs.getInt("numeric_scale")
), tableName.replace("\"", ""));
log.debug("metadata of table \"{}\" is {}", tableName, columnMetaDataList);
return columnMetaDataList.isEmpty() ? null :
columnMetaDataList.stream().collect(Collectors.toMap(ColumnMetaData::getName, Function.identity()));
}
}