imdg http://git.mfd.msk/mfd/clearing/-/issues/22 debug reload

clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/element/ControllerSettings.java

@@ -0,0 +1,25 @@
+package ru.spcex.clearing.imdg.config.element;
+
+/**
+ * Debug config
+ */
+public class ControllerSettings {
+    private String port;
+    private String contextPath;
+
+    public String getPort() {
+        return port;
+    }
+
+    public void setPort(String port) {
+        this.port = port;
+    }
+
+    public String getContextPath() {
+        return contextPath;
+    }
+
+    public void setContextPath(String contextPath) {
+        this.contextPath = contextPath;
+    }
+}

clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/config/element/ImdgSettings.java

@@ -10,6 +10,7 @@ import org.springframework.stereotype.Component;
 public class ImdgSettings {
     private HazelcastServerSettings hazelcast;
     private DatabaseSettings database;
+    private ControllerSettings debugServer;
 
     public HazelcastServerSettings getHazelcast() {
         return hazelcast;
@@ -26,4 +27,12 @@ public class ImdgSettings {
     public void setDatabase(DatabaseSettings database) {
         this.database = database;
     }
+
+    public ControllerSettings getDebugServer() {
+        return debugServer;
+    }
+
+    public void setDebugServer(ControllerSettings debugServer) {
+        this.debugServer = debugServer;
+    }
 }

clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/AbstractHazelcastLifecycleSupport.java

@@ -16,6 +16,7 @@ import ru.spcex.clearing.imdg.base.SimpleObjectMapStore;
 import ru.spcex.platform.imdg.iml.hazelcast.util.HazelcastHelper;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.*;
 
@@ -121,6 +122,77 @@ public abstract class AbstractHazelcastLifecycleSupport implements InitializingB
 
     }
 
+    public int reloadMapFromDB() {
+        log.info("Reload all from DB...");
+        long loadTime = System.currentTimeMillis();
+        int count = 0;
+
+        try {
+            List<Callable<Long>> tasks = new ArrayList<>();
+            Collection<String> mapNames = hazelcastServerInstance.getConfig().getMapConfigs().keySet();
+            for (String mapName : mapNames) {
+                tasks.add(() -> {
+                    Long maxKey = null;
+                    MapStoreConfig mapStoreConfig = hazelcastServerInstance.getConfig().getMapConfig(mapName).getMapStoreConfig();
+                    if (mapStoreConfig != null && mapStoreConfig.isEnabled()) {
+                        long start = System.currentTimeMillis();
+                        log.debug("evict map {}", mapName);
+                        IMap<Long, BusinessObject> map = hazelcastServerInstance.getMap(mapName);
+                        map.evictAll();
+                        log.debug("Load map {}", mapName);
+                        map.loadAll(false);
+                        int size = map.size();
+                        long time = System.currentTimeMillis() - start;
+                        log.debug("{} {} rows reloaded in {}ms", mapName, size, time);
+
+                        Object mapStore = mapStoreConfig.getImplementation();
+                        if (mapStore instanceof SimpleObjectMapStore) {
+                            String tableName = ((SimpleObjectMapStore) mapStore).getTableName();
+                            maxKey = map.keySet().stream().max(Long::compareTo).orElse(null); // jdbcTemplate.queryForObject("select max(id) from " + tableName, Long.class);
+                        } else if (mapStore instanceof DictionaryMapStore) {
+                            // для Dictionary не используется общий id генератор
+//                        } else if (mapStore instanceof FrontendUserSessionMapStore) {
+//                            // не используется общий id генератор
+                        } else {
+                            throw new RuntimeException("unknown map store implementation " + mapStore);
+                        }
+                        log.debug("{} max(id)={}", mapName, maxKey);
+                    }
+
+                    return maxKey;
+                });
+            }
+            long maxKey = 0L;
+            int threadCount = Runtime.getRuntime().availableProcessors();// todo config * Config.get().getRoot().getSettings().getInitHazelcastThreadMultiplier();
+            log.info("Initializing threads count = {}", threadCount);
+            ExecutorService executor = Executors.newWorkStealingPool(threadCount);
+            try {
+                List<Future<Long>> results = executor.invokeAll(tasks);
+                for (Future<Long> result : results) {
+                    Long maxKeyResult = result.get();
+                    if (maxKeyResult != null) {
+                        maxKey = Math.max(maxKey, maxKeyResult);
+                    }
+                    count++;
+                }
+            } finally {
+                executor.shutdown();
+            }
+            log.info("IDGenerator can not reinit. Max map ID {}", maxKey);
+//                plannerAllTodayMaker.makeSchedulerAllTodayMap();
+        } catch (InterruptedException | ExecutionException e) {
+            if (e instanceof InterruptedException) {
+                Thread.currentThread().interrupt();
+            }
+            throw new RuntimeException("MapStore multithreaded reload not complete.", e);
+        }
+
+//        HazelcastHelper.imdgSystem_setStorageState(true, hazelcastServerInstance);
+        loadTime = System.currentTimeMillis() - loadTime;
+        log.info("All map reload time {} ms", loadTime);
+        return count;
+    }
+
     @Override
     public void destroy() {
         hazelcastServerInstance.shutdown();

clearing-parent/imdg/src/main/java/ru/spcex/clearing/imdg/services/controller/ImdgController.java

@@ -0,0 +1,106 @@
+package ru.spcex.clearing.imdg.services.controller;
+
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import ru.spcex.clearing.imdg.config.element.ControllerSettings;
+import ru.spcex.clearing.imdg.config.element.ImdgSettings;
+import ru.spcex.clearing.imdg.services.AbstractHazelcastLifecycleSupport;
+import ru.spcex.platform.utils.log.ExceptionUtils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDateTime;
+
+@Service
+public class ImdgController implements InitializingBean, DisposableBean {
+    protected final Logger log = LoggerFactory.getLogger(getClass());
+    HttpServer server;
+
+    final ControllerSettings settings;
+    final AbstractHazelcastLifecycleSupport imdgService;
+
+    @Autowired
+    public ImdgController(AbstractHazelcastLifecycleSupport imdgService, ImdgSettings settings) {
+        this.imdgService = imdgService;
+        this.settings = settings.getDebugServer();
+    }
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        Integer port = null; // 8701
+        try {
+            if (settings == null || StringUtils.isEmpty(settings.getPort())) {
+                log.debug("Debug HTTP port not set, do not init HTTP controller service.");
+                return;
+            }
+            port = Integer.parseInt(settings.getPort().trim());
+            String url = StringUtils.isBlank(settings.getContextPath()) ? "/imdg" : settings.getContextPath().trim();
+            log.info("Controller for debug reload star at port {} with url \"{}\"", port, url);
+            this.server = HttpServer.create(new InetSocketAddress(port), 0);
+            server.createContext(url, new MyHandler());
+            server.setExecutor(null); // default
+            server.start();
+        } catch (Throwable t) {
+            log.error("Can not start debug HTTP server in port {}: {}", port, ExceptionUtils.getStackTrace(t));
+        }
+    }
+
+    @Override
+    public void destroy() throws Exception {
+        if (server != null) {
+            server.stop(100);
+            log.info("Controller for reload stop.");
+        }
+    }
+
+    class MyHandler implements HttpHandler {
+        @Override
+        public void handle(HttpExchange t) throws IOException {
+            log.info("Reload request by user \"{}\"", t.getRemoteAddress().getAddress());
+            t.sendResponseHeaders(200, 0);
+            t.setAttribute("Content-Type", "text/plain; charset=utf-8"); // or "text/html; charset=utf-8"
+            try (OutputStream os = t.getResponseBody()) {
+                // writeHead(os);
+                writeLine(os, "Wait, reload all maps from DB... " + LocalDateTime.now());
+                os.flush();
+                try {
+                    long clock = System.currentTimeMillis();
+                    int count = imdgService.reloadMapFromDB();
+                    clock = System.currentTimeMillis() - clock;
+                    writeLine(os, count + " map per " + clock + " ms");
+                } catch (Throwable e) {
+                    String msg = "Error reload: " + ExceptionUtils.getStackTrace(e);
+                    log.error(msg);
+                    writeLine(os, msg);
+                }
+                writeLine(os, "Done. " + LocalDateTime.now());
+                // writeEnd(os)
+            } //os.close();
+        }
+
+        void writeHead(OutputStream os) throws IOException {
+            //writeLine(os, "<doctype html><html><body>");
+        }
+
+        void writeLine(OutputStream os, String text) throws IOException {
+            if (text != null)
+                os.write(text.getBytes(StandardCharsets.UTF_8));
+            os.write("\n".getBytes(StandardCharsets.UTF_8));
+        }
+
+        void writeEnd(OutputStream os) throws IOException {
+            //writeLine(os, "</body></html>");
+        }
+    }
+}

clearing-parent/imdg/src/main/resources/application.properties

@@ -4,4 +4,9 @@ imdg.hazelcast.password=dev-pass
 imdg.hazelcast.cluster-members[0]=127.0.0.1
 imdg.database.login=clearing
 imdg.database.password=Aa111111
-imdg.database.url=jdbc:postgresql://10.200.200.133:5432/clearing?currentSchema=clearing_prod
+imdg.database.url=jdbc:postgresql://10.200.200.133:5432/clearing?currentSchema=clearing_prod
+#imdg.database.url=jdbc:postgresql://10.200.200.133:5432/postgres?currentSchema=clearing_tester
+
+#debug tester mode:
+#imdg.debug-server.port=8701
+#imdg.debug-server.context-path=/imdg/reload