From bcd1aa2b4073ad64d186c866ed55c854be24a201 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 12 Sep 2024 18:15:10 +0300 Subject: [PATCH] http://jira.mfd.msk:8088/browse/CLS-754 --- .../imdg/base/SimpleObjectMapStore.java | 149 ++++++++++++++++-- .../AbstractHazelcastLifecycleSupport.java | 43 ++++- .../clearing/imdg/util/ColumnMetaData.java | 40 +++++ .../imdg/util/PostgresColumnTypeUtil.java | 29 ++++ 4 files changed, 246 insertions(+), 15 deletions(-) create mode 100644 clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/ColumnMetaData.java create mode 100644 clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/PostgresColumnTypeUtil.java diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/base/SimpleObjectMapStore.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/base/SimpleObjectMapStore.java index 6a2bd0105..541726a30 100644 --- a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/base/SimpleObjectMapStore.java +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/base/SimpleObjectMapStore.java @@ -1,6 +1,23 @@ package ru.spcex.clearing.imdg.base; import com.hazelcast.core.MapStore; +import java.math.BigDecimal; +import java.sql.ParameterMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,18 +25,11 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.SqlTypeValue; import org.springframework.jdbc.core.StatementCreatorUtils; import org.springframework.lang.NonNull; +import ru.spcex.clearing.imdg.util.ColumnMetaData; +import ru.spcex.clearing.imdg.util.PostgresColumnTypeUtil; import ru.spcex.platform.classes.base.SpcexObjectBase; import ru.spcex.platform.utils.text.TextUtil; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.util.*; -import java.util.stream.Collectors; - /** * MapStore для выгрузки обычных объектов * @@ -30,6 +40,7 @@ public abstract class SimpleObjectMapStore implements protected final Logger log = LoggerFactory.getLogger(this.getClass()); protected final JdbcTemplate jdbcTemplate; protected SimpleDateFormat DB_DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd"); + protected Map columnMetaDataMap; protected SimpleObjectMapStore(JdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; @@ -149,10 +160,12 @@ public abstract class SimpleObjectMapStore implements } protected void batchInsertUpdate(@NonNull String insertStatement, @NonNull List args) { + updateMetaData(); try { int[][] ret = jdbcTemplate.batchUpdate(insertStatement, args, BATCH_SIZE, (ps, params) -> { for (int i = 0; i < params.length; i++) { Object value = params[i]; + value = normalizeValue(value, ps.getParameterMetaData(), columnMetaDataMap, i, insertStatement, params); try { StatementCreatorUtils.setParameterValue(ps, i + 1, SqlTypeValue.TYPE_UNKNOWN, value); } catch (SQLException e) { @@ -168,6 +181,112 @@ public abstract class SimpleObjectMapStore implements } } + Object normalizeValue(Object value, ParameterMetaData metaData, Map columnMetaDataMap, int i, String insertStatement, Object[] params) throws SQLException { + if (value == null) + return null; + i++; // нумерация в ParameterMetaData с 1 + final int type = metaData.getParameterType(i); + + final String fieldName = getFields()[i - 1].toLowerCase(); + if (columnMetaDataMap == null) { + log.trace("Column metadata was empty, can not verify {}.{}", getTableName(), fieldName); + } + if ((type == Types.VARCHAR || type == Types.CHAR) && value instanceof String stringValue) { + int length; + if (columnMetaDataMap == null) { + length = metaData.getPrecision(i + 1); + } else { + // в postgresql для String ParameterMetaData.getPrecision возвращает 0, по этому используется другой способ через columnMetaDataMap. + length = columnMetaDataMap.get(fieldName).getPrecision(); + } + if (length > 0 && stringValue.length() > length) { + value = stringValue.substring(0, length); + log.error("String value for insert is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]", + insertStatement, Arrays.toString(params), i, stringValue, value); + } + } else if ((type == Types.DECIMAL || type == Types.NUMERIC || type == Types.BIGINT || type == Types.INTEGER) && + (value instanceof BigDecimal || value instanceof Long || value instanceof Integer)) { + + if (value instanceof Integer && (type == Types.INTEGER || type == Types.BIGINT)) { + return value; // value ok + } + + if (value instanceof Long vl) { + if (type == Types.BIGINT) { + return value; // value ok + } + if (type == Types.INTEGER) { + if (vl > Integer.MAX_VALUE) { + value = Integer.MAX_VALUE; + log.error("Long value for insert is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]", + insertStatement, Arrays.toString(params), i, vl, value); + } + if (vl < Integer.MIN_VALUE) { + value = Integer.MIN_VALUE; + log.error("Long value for insert is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]", + insertStatement, Arrays.toString(params), i, vl, value); + } + return value; + } + } + + BigDecimal bigDecimalValue; + if (value instanceof Integer intValue) { + bigDecimalValue = BigDecimal.valueOf(intValue); + } else if (value instanceof Long longValue) { + bigDecimalValue = BigDecimal.valueOf(longValue); + } else { + bigDecimalValue = (BigDecimal) value; + } + if (type == Types.BIGINT) { + if (bigDecimalValue.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { + Object oldValue = value; + value = Long.MAX_VALUE; + log.error("Double value for insert as BIGINT is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]", + insertStatement, Arrays.toString(params), i, oldValue, value); + } else if (bigDecimalValue.compareTo(BigDecimal.valueOf(Long.MIN_VALUE)) < 0) { + Object oldValue = value; + value = Long.MIN_VALUE; + log.error("Double value for insert as BIGINT is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]", + insertStatement, Arrays.toString(params), i, oldValue, value); + } + return value; // value ok + } + if (type == Types.INTEGER) { + if (bigDecimalValue.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) > 0) { + Object oldValue = value; + value = Integer.MAX_VALUE; + log.error("Double value for insert as INTEGER is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]", + insertStatement, Arrays.toString(params), i, oldValue, value); + } else if (bigDecimalValue.compareTo(BigDecimal.valueOf(Integer.MIN_VALUE)) < 0) { + Object oldValue = value; + value = Integer.MIN_VALUE; + log.error("Double value for insert as INTEGER is too large [insertSql: {}, params: {}, i: {}, incorrect value: {}, normalized value: {}]", + insertStatement, Arrays.toString(params), i, oldValue, value); + } + return value; // value ok + } + int dbScalePartLength; + int precision; + if (columnMetaDataMap == null) { + dbScalePartLength = metaData.getScale(i); + precision = metaData.getPrecision(i); + } else { + dbScalePartLength = columnMetaDataMap.get(fieldName).getScale(); + precision = columnMetaDataMap.get(fieldName).getPrecision(); + } + if (!checkBigDecimalSize(bigDecimalValue, precision, dbScalePartLength)) { + boolean negative = bigDecimalValue.signum() < 0; + value = new BigDecimal(BigDecimal.valueOf(Math.pow(10, precision - dbScalePartLength)).longValue() - 1); + if (negative) + value = ((BigDecimal) value).negate(); + log.error("BigDecimal value for insert is too large [ insertSql: {}, params: {}, i:{}, incorrect value : {}, normalized value: {} ]", + insertStatement, Arrays.toString(params), i, bigDecimalValue, value); + } + } + return value; + } + protected Instant getInstantFromTimestamp(ResultSet rs, String column) throws SQLException { Timestamp date = rs.getTimestamp(column); return date != null ? date.toInstant() : null; @@ -191,4 +310,16 @@ public abstract class SimpleObjectMapStore implements return TextUtil.format(INSERT_TEMPLATE, params); } + + private void updateMetaData() { + if (columnMetaDataMap == null) synchronized (this) { + columnMetaDataMap = PostgresColumnTypeUtil.extractTableMetadata(jdbcTemplate, getTableName()); + } + } + + private boolean checkBigDecimalSize(BigDecimal value, int dbSize, int dbScale) { + if (value == null) + return true; + return value.precision() - value.scale() <= dbSize - dbScale; + } } diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java index 86c88164d..7672a79e3 100644 --- a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java @@ -4,6 +4,14 @@ import com.hazelcast.config.MapStoreConfig; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.core.IMap; import com.hazelcast.core.IdGenerator; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.DisposableBean; @@ -16,10 +24,6 @@ import ru.spcex.clearing.imdg.base.SimpleObjectMapStore; import ru.spcex.clearing.imdg.object.DbVersionMapStore; import ru.spcex.platform.imdg.iml.hazelcast.util.HazelcastHelper; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - //todo почистить класс public abstract class AbstractHazelcastLifecycleSupport implements InitializingBean, DisposableBean { private final Logger log = LoggerFactory.getLogger(this.getClass()); @@ -48,7 +52,7 @@ public abstract class AbstractHazelcastLifecycleSupport implements InitializingB @Override public void afterPropertiesSet() { // HazelcastCommon.loadClassesVersionsByStorage(hazelcastServerInstance, getSerialVersionUIDClasses()); -// databaseVersionCheck(); + databaseVersionCheck(); long loadTime = System.currentTimeMillis(); @@ -119,11 +123,38 @@ public abstract class AbstractHazelcastLifecycleSupport implements InitializingB HazelcastHelper.imdgSystem_setStorageState(true, hazelcastServerInstance); loadTime = System.currentTimeMillis() - loadTime; log.info("All map load time {} ms", loadTime); - } @Override public void destroy() { hazelcastServerInstance.shutdown(); } + + private void databaseVersionCheck() { + log.debug("Validating database version, expected \"{}\"", getCheckDbVersion()); + + String dbVersion = jdbcTemplate.queryForObject("SELECT version FROM db_version", String.class); + log.trace("Database version is \"{}\"", dbVersion); + if (dbVersion == null || dbVersion.isBlank()) { + log.error("Database version is empty"); + System.exit(1); + } else { + List expectedVersion = Arrays.stream(getCheckDbVersion().split("\\.")) + .map(Integer::parseInt) + .toList(); + List actualVersion = Arrays.stream(dbVersion.split("\\.")) + .map(Integer::parseInt) + .toList(); + + if (expectedVersion.size() < 2 || actualVersion.size() < 2) { + log.error("Database version have wrong format, expected = {}, actual = {}", expectedVersion, actualVersion); + System.exit(1); + } + + if (!expectedVersion.get(0).equals(actualVersion.get(0)) || expectedVersion.get(1) > actualVersion.get(1)) { + log.error("Database version is older than expected version, expected = {}, actual = {}", expectedVersion, actualVersion); + System.exit(1); + } + } + } } diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/ColumnMetaData.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/ColumnMetaData.java new file mode 100644 index 000000000..1558a4b20 --- /dev/null +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/ColumnMetaData.java @@ -0,0 +1,40 @@ +package ru.spcex.clearing.imdg.util; + +public class ColumnMetaData { + private final String name; + private final String dataType; + private final int characterMaximumLength; + private final int numericPrecision; + private final int numericScale; + + public ColumnMetaData(String name, String dataType, int characterMaximumLength, int numericPrecision, int numericScale) { + this.name = name; + this.dataType = dataType; + this.characterMaximumLength = characterMaximumLength; + this.numericPrecision = numericPrecision; + this.numericScale = numericScale; + } + + public String getName() { + return name; + } + + public String getDataType() { + return dataType; + } + + public int getPrecision() { + return switch (dataType) { + case "smallint", "integer", "real", "numeric", "bigint", "double precision" -> numericPrecision; + case "character varying" -> characterMaximumLength; + default -> 0; + }; + } + + public int getScale() { + return switch (dataType) { + case "smallint", "integer", "real", "numeric", "bigint", "double precision" -> numericScale; + default -> 0; + }; + } +} diff --git a/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/PostgresColumnTypeUtil.java b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/PostgresColumnTypeUtil.java new file mode 100644 index 000000000..1d6b7035a --- /dev/null +++ b/clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/util/PostgresColumnTypeUtil.java @@ -0,0 +1,29 @@ +package ru.spcex.clearing.imdg.util; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; + +public final class PostgresColumnTypeUtil { + static Logger log = LoggerFactory.getLogger(PostgresColumnTypeUtil.class); + static String SELECT_METADATA_CASE_SENS = "SELECT * FROM information_schema.columns WHERE table_name=?"; + static String SELECT_METADATA_CASE_INSENS = "SELECT * FROM information_schema.columns WHERE table_name ilike ?"; + + public static Map extractTableMetadata(JdbcTemplate template, String tableName) { + List columnMetaDataList = template.query(tableName.contains("\"") ? SELECT_METADATA_CASE_SENS + : SELECT_METADATA_CASE_INSENS, (rs, rn) -> new ColumnMetaData( + rs.getString("column_name"), + rs.getString("data_type"), + rs.getInt("character_maximum_length"), + rs.getInt("numeric_precision"), + rs.getInt("numeric_scale") + ), tableName.replace("\"", "")); + log.debug("metadata of table \"{}\" is {}", tableName, columnMetaDataList); + return columnMetaDataList.isEmpty() ? null : + columnMetaDataList.stream().collect(Collectors.toMap(ColumnMetaData::getName, Function.identity())); + } +}