diff --git a/packaging/pom.xml b/packaging/pom.xml index 46949bd66b7f..5af8c8d91253 100644 --- a/packaging/pom.xml +++ b/packaging/pom.xml @@ -150,6 +150,10 @@ http://(www\.)?antlr.org/license.html http://www.antlr.org/license.html + + https?://www\.eclipse\.org/legal/epl-2\.0/? + https://www.eclipse.org/legal/epl-2.0 + @@ -157,6 +161,7 @@ \Qhttps://aws.amazon.com/apache2.0\E \Qhttps://opensource.org/licenses/Apache-2.0\E \Qhttp://www.apache.org/licenses/\E + \Qhttps://repository.jboss.org/licenses/apache-2.0.txt\E https?://(www\.)?opensource.org/licenses/bsd-license.php diff --git a/pom.xml b/pom.xml index 0ea93db9413f..49ddeabda287 100644 --- a/pom.xml +++ b/pom.xml @@ -235,9 +235,9 @@ 2.1.1 2.4.0 - 5.3.39 - 2.7.18 - 2.4.4 + 6.2.12 + 3.4.11 + 3.3.4 2025-01-01T00:00:00Z 26.0.6 11.28 diff --git a/standalone-metastore/metastore-rest-catalog/pom.xml b/standalone-metastore/metastore-rest-catalog/pom.xml index d987f7cce972..44c84ce15abc 100644 --- a/standalone-metastore/metastore-rest-catalog/pom.xml +++ b/standalone-metastore/metastore-rest-catalog/pom.xml @@ -31,6 +31,12 @@ hive-standalone-metastore-server ${hive.version} provided + + + javax.servlet + javax.servlet-api + + org.apache.hive @@ -70,6 +76,12 @@ micrometer-registry-prometheus 1.9.17 + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + provided + org.apache.hive diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java index d21f239f3416..cdb3e3897acd 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -import javax.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServlet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.ServletSecurity; import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType; diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java index 6140f40b2de5..76a971d43f70 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java @@ -25,9 +25,9 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.rest.HMSCatalogAdapter.Route; import org.apache.iceberg.rest.HTTPRequest.HTTPMethod; diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/standalone/IcebergCatalogConfiguration.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/standalone/IcebergCatalogConfiguration.java index 7f30b3221a90..7296f6fa513d 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/standalone/IcebergCatalogConfiguration.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/standalone/IcebergCatalogConfiguration.java @@ -17,8 +17,6 @@ */ package org.apache.iceberg.rest.standalone; -import javax.servlet.http.HttpServlet; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.ServletServerBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -54,14 +52,14 @@ public Configuration hadoopConfiguration(ApplicationArguments args) { } @Bean - public ServletRegistrationBean restCatalogServlet(Configuration conf) { + public ServletRegistrationBean restCatalogServlet(Configuration conf) { return createRestCatalogServlet(conf); } /** * Creates the REST Catalog servlet registration. Shared by production config and tests. */ - public static ServletRegistrationBean createRestCatalogServlet(Configuration conf) { + public static ServletRegistrationBean createRestCatalogServlet(Configuration conf) { String servletPath = MetastoreConf.getVar(conf, ConfVars.ICEBERG_CATALOG_SERVLET_PATH); if (servletPath == null || servletPath.isEmpty()) { servletPath = DEFAULT_SERVLET_PATH; @@ -80,9 +78,8 @@ public static ServletRegistrationBean createRestCatalogServlet(Conf if (descriptor == null || descriptor.getServlet() == null) { throw new IllegalStateException("Failed to create Iceberg REST Catalog servlet"); } - - ServletRegistrationBean registration = - new ServletRegistrationBean<>(descriptor.getServlet(), "/" + servletPath + "/*"); + ServletRegistrationBean registration = + new ServletRegistrationBean(descriptor.getServlet(), "/" + servletPath + "/*"); registration.setName("IcebergRESTCatalog"); registration.setLoadOnStartup(1); diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 64926eaba1c9..cdbaf4184a60 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -288,6 +288,7 @@ org.apache.thrift libthrift + 0.20.0 org.datanucleus @@ -359,17 +360,23 @@ org.eclipse.jetty jetty-util - ${jetty.version} + 11.0.25 org.eclipse.jetty jetty-server - ${jetty.version} + 11.0.25 org.eclipse.jetty jetty-servlet - ${jetty.version} + 11.0.25 + + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + provided org.springframework diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 2ffe2351dee4..edab9f4478fc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -99,9 +99,9 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import javax.servlet.Servlet; -import javax.servlet.ServletRequestEvent; -import javax.servlet.ServletRequestListener; +import jakarta.servlet.Servlet; +import jakarta.servlet.ServletRequestEvent; +import jakarta.servlet.ServletRequestListener; /** * TODO:pc remove application logic to a separate interface. */ @@ -382,7 +382,7 @@ public void setThreadFactory(ThreadFactory threadFactory) { final HttpConnectionFactory http = new HttpConnectionFactory(httpServerConf); - final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(conf); + final SslContextFactory.Server sslContextFactory = ServletSecurity.createSslContextFactory(conf); if (sslContextFactory != null) { connector = new ServerConnector(server, sslContextFactory, http); } else { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java index adea578f2392..e1de388d44de 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java @@ -34,11 +34,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletException; -import javax.servlet.ServletOutputStream; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index c5a83ef5d7ff..bb789841238f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -33,22 +33,18 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.pac4j.core.context.JEEContext; -import org.pac4j.core.credentials.TokenCredentials; -import org.pac4j.core.credentials.extractor.BearerAuthExtractor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Enumeration; -import java.util.Optional; /** * Secures servlet processing. @@ -305,10 +301,15 @@ private String extractUserName(HttpServletRequest request, HttpServletResponse r */ private String extractBearerToken(HttpServletRequest request, HttpServletResponse response) { - BearerAuthExtractor extractor = new BearerAuthExtractor(); - Optional tokenCredentials = extractor.extract(new JEEContext( - request, response)); - return tokenCredentials.map(TokenCredentials::getToken).orElse(null); + String authorization = request.getHeader("Authorization"); + if (authorization == null) { + return null; + } + if (authorization.regionMatches(true, 0, "Bearer ", 0, 7)) { + String token = authorization.substring(7).trim(); + return token.isEmpty() ? null : token; + } + return null; } /** @@ -338,7 +339,7 @@ static void loginServerPrincipal(Configuration conf) throws IOException { * @return null if no ssl in config, an instance otherwise * @throws IOException if getting password fails */ - static SslContextFactory createSslContextFactory(Configuration conf) throws IOException { + static SslContextFactory.Server createSslContextFactory(Configuration conf) throws IOException { final boolean useSsl = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_SSL); if (!useSsl) { return null; @@ -359,7 +360,7 @@ static SslContextFactory createSslContextFactory(Configuration conf) throws IOEx if (LOG.isInfoEnabled()) { LOG.info("HTTP Server SSL: adding excluded protocols: {}", Arrays.toString(excludedProtocols)); } - SslContextFactory factory = new SslContextFactory.Server(); + SslContextFactory.Server factory = new SslContextFactory.Server(); factory.addExcludeProtocols(excludedProtocols); if (LOG.isInfoEnabled()) { LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = {}", diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java index 67f9c9961b6c..c4bc5d1a1e2d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java @@ -38,8 +38,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.Servlet; -import javax.servlet.http.HttpServlet; +import jakarta.servlet.Servlet; +import jakarta.servlet.http.HttpServlet; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -173,7 +173,7 @@ private Server createServer() { * @param port The port to bind the connector to * @return The created ServerConnector */ - private ServerConnector createConnector(Server server, SslContextFactory sslContextFactory, int port) { + private ServerConnector createConnector(Server server, SslContextFactory.Server sslContextFactory, int port) { final ServerConnector connector; HttpConfiguration httpConf = new HttpConfiguration(); // Do not leak information @@ -236,7 +236,7 @@ public Server startServer() throws Exception { } final Server server = createServer(); // create the connectors - final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(configuration); + final SslContextFactory.Server sslContextFactory = ServletSecurity.createSslContextFactory(configuration); final ServerConnector[] connectors = new ServerConnector[size]; final ServletContextHandler[] handlers = new ServletContextHandler[size]; Iterator> it = handlersMap.entrySet().iterator(); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java index 13559fc384db..c13848c1acb6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java @@ -15,7 +15,7 @@ package org.apache.hadoop.hive.metastore.auth; import java.util.Optional; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; /* Encapsulates any exceptions thrown by HiveMetastore server diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/TokenIntrospectionAuthenticator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/TokenIntrospectionAuthenticator.java index f46910647367..43bbe608f972 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/TokenIntrospectionAuthenticator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/TokenIntrospectionAuthenticator.java @@ -36,7 +36,7 @@ import java.time.Clock; import java.time.Duration; import java.util.List; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; import org.checkerframework.checker.index.qual.NonNegative; import org.checkerframework.checker.nullness.qual.NonNull; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 4146661f6c5c..9672d230e335 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -80,7 +80,6 @@ import org.apache.hadoop.hive.metastore.utils.StringUtils; import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator; import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator.ObjectEstimator; -import org.eclipse.jetty.util.ConcurrentHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,7 +115,7 @@ public class SharedCache { private AtomicLong cacheUpdateCount = new AtomicLong(0); private long maxCacheSizeInBytes = -1; private HashMap, ObjectEstimator> sizeEstimators = null; - private Set tableToUpdateSize = new ConcurrentHashSet<>(); + private Set tableToUpdateSize = ConcurrentHashMap.newKeySet(); private ScheduledExecutorService executor = null; private Map tableSizeMap = null; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/OnRenameFunction.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/OnRenameFunction.java index 484e0df9e4c2..44be47b36365 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/OnRenameFunction.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/OnRenameFunction.java @@ -149,7 +149,12 @@ public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaExcepti // caught and either swallowed or wrapped in MetaException. Also, only a single test fails without this block: // org.apache.hadoop.hive.metastore.client.TestDatabases.testAlterDatabaseNotNullableFields // It may worth investigate if this catch block is really needed. - if (e.getMessage() != null && e.getMessage().contains("does not exist")) { + String msg = e.getMessage(); + Throwable cause = e.getCause(); + boolean isMissingTable = (msg != null && msg.contains("does not exist")) || + (cause != null && cause.getMessage() != null && cause.getMessage().contains("does not exist")); + + if (isMissingTable) { LOG.warn("Cannot perform {} since metastore table does not exist", callSig); } else { throw new MetaException("Unable to " + callSig + ":" + org.apache.hadoop.util.StringUtils.stringifyException(e)); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java index e752ef592e26..e4f314b2b7c1 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java @@ -31,11 +31,11 @@ import java.util.IdentityHashMap; import java.util.Map; import java.util.function.Function; -import javax.servlet.Servlet; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.Servlet; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.eclipse.jetty.server.Server; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSHttpClientSslTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSHttpClientSslTest.java index 4a276ef87bab..e971c2da1605 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSHttpClientSslTest.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSHttpClientSslTest.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.TreeMap; import javax.net.ssl.SSLContext; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSJsonClientSslTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSJsonClientSslTest.java index a8d52aa17132..2b7ace7197a4 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSJsonClientSslTest.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSJsonClientSslTest.java @@ -35,7 +35,7 @@ import java.util.TreeMap; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.PropertyServlet; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; diff --git a/standalone-metastore/packaging/pom.xml b/standalone-metastore/packaging/pom.xml index 4fb9de38f3ea..ec16c4a443db 100644 --- a/standalone-metastore/packaging/pom.xml +++ b/standalone-metastore/packaging/pom.xml @@ -96,6 +96,10 @@ http://(www\.)?antlr.org/license.html http://www.antlr.org/license.html + + https?://www\.eclipse\.org/legal/epl-2\.0/? + https://www.eclipse.org/legal/epl-2.0 + @@ -103,6 +107,7 @@ \Qhttps://aws.amazon.com/apache2.0\E \Qhttps://opensource.org/licenses/Apache-2.0\E \Qhttp://www.apache.org/licenses/\E + \Qhttps://repository.jboss.org/licenses/apache-2.0.txt\E https?://(www\.)?opensource.org/licenses/bsd-license.php diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index dd0650312ce2..9869a00a0a65 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -126,9 +126,9 @@ 1.3.2 26.0.6 - 5.3.39 - 2.7.18 - 2.4.4 + 6.2.12 + 3.4.11 + 3.3.4 1.21.3 you-must-set-this-to-run-thrift