From 33fcf90954f9bb2bd76191b6e90b62aff39a9453 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 15 May 2026 18:19:29 +0800 Subject: [PATCH] [core]: bound jdbc waits on vip failover Add default MySQL JDBC connect/read timeouts and TCP keepalive to management-node datasource URLs so stale VIP connections fail and recover instead of waiting indefinitely in Connector/J socket reads. Constraint: 5.5.22 deployments may configure datasource URLs explicitly through zstack.properties, so explicit DbFacadeDataSource and RESTApiDataSource URLs must also receive defaults when they do not already set them. Rejected: Only changing XML fallback URLs | zstack.properties datasource URLs bypass the XML fallback in deployed environments. Confidence: high Scope-risk: moderate Directive: Do not remove the JDBC timeout defaults without re-testing keepalived VIP movement against a running MN. Tested: JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 mvn -pl core -DskipTests install Tested: JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 mvn -pl plugin/kvm -DskipTests install Tested: JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 mvn -Dtest=org.zstack.test.core.TestDefaultDbJdbcUrl -Dsurefire.useFile=false test Tested: Manual 172.20.1.164/172.20.1.165 keepalived VIP failover, 40/40 zstack-cli queries succeeded with bounded post-failover delay Not-tested: Full CI suite Resolves: ZSTAC-0 Change-Id: Ie551c05a6b646c192f01e47fdda902b388cc2b04 --- conf/springConfigXml/DatabaseFacade.xml | 4 +- conf/springConfigXml/RESTFacade.xml | 2 +- .../main/java/org/zstack/core/Platform.java | 99 +++++++++++---- .../core/db/DatabaseGlobalProperty.java | 6 + .../test/core/TestDefaultDbJdbcUrl.java | 116 ++++++++++++++++++ 5 files changed, 203 insertions(+), 24 deletions(-) create mode 100644 test/src/test/java/org/zstack/test/core/TestDefaultDbJdbcUrl.java diff --git a/conf/springConfigXml/DatabaseFacade.xml b/conf/springConfigXml/DatabaseFacade.xml index d7afee98688..c88521f165c 100755 --- a/conf/springConfigXml/DatabaseFacade.xml +++ b/conf/springConfigXml/DatabaseFacade.xml @@ -20,7 +20,7 @@ - + @@ -42,7 +42,7 @@ - + diff --git a/conf/springConfigXml/RESTFacade.xml b/conf/springConfigXml/RESTFacade.xml index 044e4e8bb50..73c216e5e0c 100755 --- a/conf/springConfigXml/RESTFacade.xml +++ b/conf/springConfigXml/RESTFacade.xml @@ -32,7 +32,7 @@ - + diff --git a/core/src/main/java/org/zstack/core/Platform.java b/core/src/main/java/org/zstack/core/Platform.java index 78b184d12e7..01d685dc78d 100755 --- a/core/src/main/java/org/zstack/core/Platform.java +++ b/core/src/main/java/org/zstack/core/Platform.java @@ -312,39 +312,22 @@ private static void writePidFile() throws IOException { private static void prepareDefaultDbProperties() { if (DatabaseGlobalProperty.DbUrl != null) { String dbUrl = DatabaseGlobalProperty.DbUrl; - if (dbUrl.endsWith("/")) { - dbUrl = dbUrl.substring(0, dbUrl.length()-1); - } if (getGlobalProperty("DbFacadeDataSource.jdbcUrl") == null) { - String url; - if (dbUrl.contains("{database}")) { - url = ln(dbUrl).formatByMap( - map(e("database", "zstack")) - ); - url = url.trim(); - } else { - url = String.format("%s/zstack", dbUrl); - } + String url = buildDbJdbcUrl(dbUrl, "zstack"); System.setProperty("DbFacadeDataSource.jdbcUrl", url); logger.debug(String.format("default DbFacadeDataSource.jdbcUrl to DB.url [%s]", url)); } if (getGlobalProperty("RESTApiDataSource.jdbcUrl") == null) { - String url; - if (dbUrl.contains("{database}")) { - url = ln(dbUrl).formatByMap( - map(e("database", "zstack_rest")) - ); - url = url.trim(); - } else { - url = String.format("%s/zstack_rest", dbUrl); - } + String url = buildDbJdbcUrl(dbUrl, "zstack_rest"); System.setProperty("RESTApiDataSource.jdbcUrl", url); logger.debug(String.format("default RESTApiDataSource.jdbcUrl to DB.url [%s]", url)); } } + prepareDefaultDbJdbcUrl("DbFacadeDataSource.jdbcUrl"); + prepareDefaultDbJdbcUrl("RESTApiDataSource.jdbcUrl"); if (DatabaseGlobalProperty.DbUser != null) { if (getGlobalProperty("DbFacadeDataSource.user") == null) { System.setProperty("DbFacadeDataSource.user", DatabaseGlobalProperty.DbUser); @@ -395,6 +378,80 @@ private static void prepareDefaultDbProperties() { } } + private static void prepareDefaultDbJdbcUrl(String propertyName) { + String jdbcUrl = getGlobalProperty(propertyName); + if (StringUtils.isBlank(jdbcUrl)) { + return; + } + + String defaultedJdbcUrl = appendDefaultDbJdbcParameters(jdbcUrl); + if (!jdbcUrl.equals(defaultedJdbcUrl)) { + System.setProperty(propertyName, defaultedJdbcUrl); + logger.debug(String.format("append default JDBC parameters to %s [%s]", propertyName, defaultedJdbcUrl)); + } + } + + private static String buildDbJdbcUrl(String dbUrl, String database) { + String url; + if (dbUrl.contains("{database}")) { + url = ln(dbUrl).formatByMap( + map(e("database", database)) + ).trim(); + } else { + url = appendDatabaseToJdbcUrl(dbUrl, database); + } + + return appendDefaultDbJdbcParameters(url); + } + + private static String appendDatabaseToJdbcUrl(String dbUrl, String database) { + String trimmed = dbUrl.trim(); + int queryIndex = trimmed.indexOf('?'); + String base = queryIndex >= 0 ? trimmed.substring(0, queryIndex) : trimmed; + String query = queryIndex >= 0 ? trimmed.substring(queryIndex) : ""; + + while (base.endsWith("/")) { + base = base.substring(0, base.length() - 1); + } + + return String.format("%s/%s%s", base, database, query); + } + + private static String appendDefaultDbJdbcParameters(String jdbcUrl) { + String url = jdbcUrl; + url = appendJdbcParameterIfAbsent(url, "connectTimeout", DatabaseGlobalProperty.DbConnectTimeout); + url = appendJdbcParameterIfAbsent(url, "socketTimeout", DatabaseGlobalProperty.DbSocketTimeout); + url = appendJdbcParameterIfAbsent(url, "tcpKeepAlive", DatabaseGlobalProperty.DbTcpKeepAlive); + return url; + } + + private static String appendJdbcParameterIfAbsent(String jdbcUrl, String key, String value) { + if (StringUtils.isBlank(value) || hasJdbcParameter(jdbcUrl, key)) { + return jdbcUrl; + } + + String separator = jdbcUrl.contains("?") ? "&" : "?"; + return String.format("%s%s%s=%s", jdbcUrl, separator, key, value.trim()); + } + + private static boolean hasJdbcParameter(String jdbcUrl, String key) { + int queryIndex = jdbcUrl.indexOf('?'); + if (queryIndex < 0 || queryIndex == jdbcUrl.length() - 1) { + return false; + } + + String query = jdbcUrl.substring(queryIndex + 1); + for (String parameter : query.split("&")) { + int equalsIndex = parameter.indexOf('='); + String parameterKey = equalsIndex >= 0 ? parameter.substring(0, equalsIndex) : parameter; + if (key.equalsIgnoreCase(parameterKey.trim())) { + return true; + } + } + + return false; + } + private static void prepareHibernateSearchProperties() { if (!SearchGlobalProperty.SearchAutoRegister) { System.setProperty("Search.autoRegister", "false"); diff --git a/core/src/main/java/org/zstack/core/db/DatabaseGlobalProperty.java b/core/src/main/java/org/zstack/core/db/DatabaseGlobalProperty.java index 9354a5235a8..121ca52fa4a 100755 --- a/core/src/main/java/org/zstack/core/db/DatabaseGlobalProperty.java +++ b/core/src/main/java/org/zstack/core/db/DatabaseGlobalProperty.java @@ -22,6 +22,12 @@ public class DatabaseGlobalProperty { public static String DbIdleConnectionTestPeriod; @GlobalProperty(name="DB.maxIdleTime", defaultValue = "60") public static String DbMaxIdleTime; + @GlobalProperty(name="DB.connectTimeout", defaultValue = "5000") + public static String DbConnectTimeout; + @GlobalProperty(name="DB.socketTimeout", defaultValue = "60000") + public static String DbSocketTimeout; + @GlobalProperty(name="DB.tcpKeepAlive", defaultValue = "true") + public static String DbTcpKeepAlive; @GlobalProperty(name="DB.glock.waitTimeout", defaultValue = "28800") public static Long GLockWaitTimeout; @GlobalProperty(name="RESTFacade.hostname") diff --git a/test/src/test/java/org/zstack/test/core/TestDefaultDbJdbcUrl.java b/test/src/test/java/org/zstack/test/core/TestDefaultDbJdbcUrl.java new file mode 100644 index 00000000000..52d7ad0e6fb --- /dev/null +++ b/test/src/test/java/org/zstack/test/core/TestDefaultDbJdbcUrl.java @@ -0,0 +1,116 @@ +package org.zstack.test.core; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.zstack.core.Platform; +import org.zstack.core.db.DatabaseGlobalProperty; + +import java.lang.reflect.Method; + +import static org.junit.Assert.assertEquals; + +public class TestDefaultDbJdbcUrl { + private String oldDbUrl; + private String oldConnectTimeout; + private String oldSocketTimeout; + private String oldTcpKeepAlive; + private String oldDbFacadeJdbcUrl; + private String oldRestApiJdbcUrl; + + @Before + public void setUp() { + oldDbUrl = DatabaseGlobalProperty.DbUrl; + oldConnectTimeout = DatabaseGlobalProperty.DbConnectTimeout; + oldSocketTimeout = DatabaseGlobalProperty.DbSocketTimeout; + oldTcpKeepAlive = DatabaseGlobalProperty.DbTcpKeepAlive; + oldDbFacadeJdbcUrl = System.getProperty("DbFacadeDataSource.jdbcUrl"); + oldRestApiJdbcUrl = System.getProperty("RESTApiDataSource.jdbcUrl"); + + DatabaseGlobalProperty.DbConnectTimeout = "5000"; + DatabaseGlobalProperty.DbSocketTimeout = "60000"; + DatabaseGlobalProperty.DbTcpKeepAlive = "true"; + System.clearProperty("DbFacadeDataSource.jdbcUrl"); + System.clearProperty("RESTApiDataSource.jdbcUrl"); + } + + @After + public void tearDown() { + DatabaseGlobalProperty.DbUrl = oldDbUrl; + DatabaseGlobalProperty.DbConnectTimeout = oldConnectTimeout; + DatabaseGlobalProperty.DbSocketTimeout = oldSocketTimeout; + DatabaseGlobalProperty.DbTcpKeepAlive = oldTcpKeepAlive; + restoreProperty("DbFacadeDataSource.jdbcUrl", oldDbFacadeJdbcUrl); + restoreProperty("RESTApiDataSource.jdbcUrl", oldRestApiJdbcUrl); + } + + @Test + public void testAppendTimeoutsToDefaultDbUrl() throws Exception { + assertEquals( + "jdbc:mysql://172.20.0.37:3306/zstack?connectTimeout=5000&socketTimeout=60000&tcpKeepAlive=true", + buildDbJdbcUrl("jdbc:mysql://172.20.0.37:3306", "zstack") + ); + } + + @Test + public void testAppendTimeoutsAfterExistingQuery() throws Exception { + assertEquals( + "jdbc:mysql://172.20.0.37:3306/zstack?useSSL=false&connectTimeout=5000&socketTimeout=60000&tcpKeepAlive=true", + buildDbJdbcUrl("jdbc:mysql://172.20.0.37:3306?useSSL=false", "zstack") + ); + } + + @Test + public void testDoNotOverrideExplicitTimeouts() throws Exception { + assertEquals( + "jdbc:mysql://172.20.0.37:3306/zstack?socketTimeout=10000&connectTimeout=1000&tcpKeepAlive=false", + buildDbJdbcUrl("jdbc:mysql://172.20.0.37:3306?socketTimeout=10000&connectTimeout=1000&tcpKeepAlive=false", "zstack") + ); + } + + @Test + public void testTemplateDbUrlKeepsQueryInPlace() throws Exception { + assertEquals( + "jdbc:mysql://172.20.0.37:3306/zstack_rest?useSSL=false&connectTimeout=5000&socketTimeout=60000&tcpKeepAlive=true", + buildDbJdbcUrl("jdbc:mysql://172.20.0.37:3306/{database}?useSSL=false", "zstack_rest") + ); + } + + @Test + public void testAppendTimeoutsToExplicitDatasourceJdbcUrls() throws Exception { + DatabaseGlobalProperty.DbUrl = null; + System.setProperty("DbFacadeDataSource.jdbcUrl", "jdbc:mysql://172.20.0.37:3306/zstack"); + System.setProperty("RESTApiDataSource.jdbcUrl", "jdbc:mysql://172.20.0.37:3306/zstack_rest?useSSL=false"); + + prepareDefaultDbProperties(); + + assertEquals( + "jdbc:mysql://172.20.0.37:3306/zstack?connectTimeout=5000&socketTimeout=60000&tcpKeepAlive=true", + System.getProperty("DbFacadeDataSource.jdbcUrl") + ); + assertEquals( + "jdbc:mysql://172.20.0.37:3306/zstack_rest?useSSL=false&connectTimeout=5000&socketTimeout=60000&tcpKeepAlive=true", + System.getProperty("RESTApiDataSource.jdbcUrl") + ); + } + + private String buildDbJdbcUrl(String dbUrl, String database) throws Exception { + Method method = Platform.class.getDeclaredMethod("buildDbJdbcUrl", String.class, String.class); + method.setAccessible(true); + return (String) method.invoke(null, dbUrl, database); + } + + private void prepareDefaultDbProperties() throws Exception { + Method method = Platform.class.getDeclaredMethod("prepareDefaultDbProperties"); + method.setAccessible(true); + method.invoke(null); + } + + private void restoreProperty(String key, String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + } +}