diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java index 07964343429..03b7996685b 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java @@ -34,12 +34,12 @@ public abstract class AbstractAzureBlobStoreBackend extends AbstractSharedBackend { - protected abstract DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options); - protected abstract DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException; - protected abstract void setHttpDownloadURIExpirySeconds(int seconds); - protected abstract void setHttpUploadURIExpirySeconds(int seconds); - protected abstract void setHttpDownloadURICacheSize(int maxSize); - protected abstract URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions); + public abstract DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options); + public abstract DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException; + public abstract void setHttpDownloadURIExpirySeconds(int seconds); + public abstract void setHttpUploadURIExpirySeconds(int seconds); + public abstract void setHttpDownloadURICacheSize(int maxSize); + public abstract URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions); public abstract void setProperties(final Properties properties); } diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainer.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainer.java new file mode 100644 index 00000000000..bfdeafd8d15 --- /dev/null +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainer.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; + +import java.io.InputStream; +import java.time.Instant; + +import org.jetbrains.annotations.NotNull; + +public interface AzureBlobContainer extends AutoCloseable { + void createIfNotExists() throws Exception; + void delete() throws Exception; + boolean deleteIfExists() throws Exception; + boolean exists() throws Exception; + @NotNull + String getName(); + @NotNull + String getContainerUri(); + void uploadBlockBlob(@NotNull String name, @NotNull InputStream input, long length) throws Exception; + @NotNull + String generateSharedAccessSignature(@NotNull Instant expiry) throws Exception; + + @Override + default void close() throws Exception { + // Most implementations do not own extra resources. + } +} diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainers.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainers.java new file mode 100644 index 00000000000..b88c29e4a58 --- /dev/null +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainers.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; + +import java.util.Optional; +import java.util.Properties; + +import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobContainerProviderV12; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobContainerV12; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobContainerProviderV8; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobContainerV8; + +public final class AzureBlobContainers { + private AzureBlobContainers() { + } + + public static boolean deleteIfExists(Properties properties) throws DataStoreException { + try (AzureBlobContainer container = getReference(properties)) { + return container.deleteIfExists(); + } catch (DataStoreException e) { + throw e; + } catch (Exception e) { + throw new DataStoreException(e); + } + } + + public static AzureBlobContainer create(Properties properties) throws DataStoreException { + try { + AzureBlobContainer container = getReference(properties); + container.createIfNotExists(); + return container; + } catch (DataStoreException e) { + throw e; + } catch (Exception e) { + throw new DataStoreException(e); + } + } + + public static Optional get(Properties properties) throws DataStoreException { + try { + AzureBlobContainer container = getReference(properties); + if (container.exists()) { + return Optional.of(container); + } + container.close(); + return Optional.empty(); + } catch (DataStoreException e) { + throw e; + } catch (Exception e) { + throw new DataStoreException(e); + } + } + + public static AzureBlobContainer getReference(Properties properties) throws DataStoreException { + AzureSdkVersion version = AzureSdkVersion.resolve(properties); + if (version == AzureSdkVersion.V12) { + AzureBlobContainerProviderV12 provider = AzureBlobContainerProviderV12.Builder + .builder(properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME)) + .initializeWithProperties(properties) + .build(); + return new AzureBlobContainerV12(provider.getBlobContainer(null, properties)); + } + AzureBlobContainerProviderV8 provider = AzureBlobContainerProviderV8.Builder + .builder(properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME)) + .initializeWithProperties(properties) + .build(); + return new AzureBlobContainerV8(provider.getBlobContainer(), provider); + } +} diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java index c13a5fc0855..b692f88b425 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java @@ -1,233 +1,41 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +/** + * Shared Azure configuration property name constants used by consumer modules + * ({@code oak-run-commons}, {@code oak-jcr}, etc.) to configure {@link AzureDataStore}. + *

+ * Property names are intentionally literal so this shared class does not force either + * version-specific SDK package to be loaded. + */ public final class AzureConstants { - /** - * Azure storage account name - */ - public static final String AZURE_STORAGE_ACCOUNT_NAME = "accessKey"; + public static final String AZURE_V12_ENABLED_PROPERTY = "blob.azure.v12.enabled"; - /** - * Azure storage account key - */ + public static final String AZURE_STORAGE_ACCOUNT_NAME = "accessKey"; public static final String AZURE_STORAGE_ACCOUNT_KEY = "secretKey"; - - /** - * Azure connection string (overrides {@link #AZURE_SAS} and {@link #AZURE_BLOB_ENDPOINT}). - */ public static final String AZURE_CONNECTION_STRING = "azureConnectionString"; - - /** - * Azure shared access signature token - */ public static final String AZURE_SAS = "azureSas"; - - /** - * Azure active directory - */ public static final String AZURE_TENANT_ID = "tenantId"; - - /** - * Azure service principal id - */ public static final String AZURE_CLIENT_ID = "clientId"; - - /** - * Azure service principal password - */ public static final String AZURE_CLIENT_SECRET = "clientSecret"; - - /** - * Azure blob endpoint - */ public static final String AZURE_BLOB_ENDPOINT = "azureBlobEndpoint"; - - /** - * Azure blob storage container name - */ public static final String AZURE_BLOB_CONTAINER_NAME = "container"; - /** - * Azure create container if doesn't exist - */ - public static final String AZURE_CREATE_CONTAINER = "azureCreateContainer"; - - /** - * Azure blob storage request timeout - */ - public static final String AZURE_BLOB_REQUEST_TIMEOUT = "socketTimeout"; - - /** - * Azure blob storage maximum retries per request - */ - public static final String AZURE_BLOB_MAX_REQUEST_RETRY = "maxErrorRetry"; - - /** - * Azure blob storage maximum connections per operation (default 1) - */ - public static final String AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION = "maxConnections"; - - /** - * Azure blob storage enable fallback to secondary location for reads - */ - public static final String AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME = "enableSecondaryLocation"; - - /** - * Is reading from the secondary location enabled by default - */ - public static final boolean AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT = false; - - /** - * Proxy host - */ - public static final String PROXY_HOST = "proxyHost"; - - /** - * Proxy port - */ - public static final String PROXY_PORT = "proxyPort"; - - /** - * TTL for presigned HTTP upload URIs - default is 0 (disabled) - */ - public static final String PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS = "presignedHttpUploadURIExpirySeconds"; - - /** - * TTL for presigned HTTP download URIs - default is 0 (disabled) - */ - public static final String PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS = "presignedHttpDownloadURIExpirySeconds"; - - /** - * Maximum size of presigned HTTP download URI cache - default is 0 (no cache) - */ - public static final String PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE = "presignedHttpDownloadURICacheMaxSize"; - - /** - * Boolean flag to allow disabling of verification check on download URI - * generation. Default is true (the existence check is performed). - * - * Some installations may prefer to disable async uploads, in which case it - * is possible to disable the existence check and thus greatly speed up the - * generation of presigned download URIs. See OAK-7998 which describes why - * the existence check was added to understand how async uploading relates - * to this feature. - */ - public static final String PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS = "presignedHttpDownloadURIVerifyExists"; - - /** - * Domain name to use for direct downloads instead of the default Azure blob storage domain. - * This is usually used when an installation has configured a CDN domain for binary downloads. - */ - public static final String PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpDownloadURIDomainOverride"; - - /** - * Domain name to use for direct uploads instead of the default Azure blob storage domain. - * This is usually used when an installation has configured a CDN domain for binary uploads. - */ - public static final String PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpUploadURIDomainOverride"; - - /** - * Property to enable/disable creation of reference secret on init. - */ - public static final String AZURE_REF_ON_INIT = "refOnInit"; - - /** - * Directory name for storing metadata files in the blob storage - */ - public static final String AZURE_BlOB_META_DIR_NAME = "META"; - - /** - * Key prefix for metadata entries, includes trailing slash for directory structure - */ - public static final String AZURE_BLOB_META_KEY_PREFIX = AZURE_BlOB_META_DIR_NAME + "/"; - - /** - * Key name for storing blob reference information - */ - public static final String AZURE_BLOB_REF_KEY = "reference.key"; - - /** - * Key name for storing last modified timestamp metadata - */ - public static final String AZURE_BLOB_LAST_MODIFIED_KEY = "lastModified"; - - /** - * Threshold size (8 MiB) above which streams are buffered to disk during upload operations - */ - public static final long AZURE_BLOB_BUFFERED_STREAM_THRESHOLD = 8L * 1024L * 1024L; - - /** - * Minimum part size (256 KiB) required for Azure Blob Storage multipart uploads - */ - public static final long AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE = 256L * 1024L; - - /** - * Maximum part size (4000 MiB / 4 GiB) allowed by Azure Blob Storage for multipart uploads - */ - public static final long AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE = 4000L * 1024L * 1024L; - - /** - * Maximum size (256 MiB) for single PUT operations in Azure Blob Storage - */ - public static final long AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE = 256L * 1024L * 1024L; - - /** - * Maximum total binary size (~190.7 TiB) that can be uploaded to Azure Blob Storage - */ - public static final long AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE = 190L * 1024L * 1024L * 1024L * 1024L; - - /** - * Maximum number of blocks (50,000) allowed per blob in Azure Blob Storage - */ - public static final int AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS = 50000; - - /** - * Maximum number of attempts to generate a unique record identifier - */ - public static final int AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES = 10; - - /** - * Default number of concurrent requests (5) for optimal performance with Azure SDK 12 - */ - public static final int AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT = 5; - - /** - * Maximum recommended concurrent requests (10) for optimal performance with Azure SDK 12 - */ - public static final int AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT = 10; - - /** - * Maximum block size (100 MiB) supported by Azure Blob Storage SDK 12 - */ - public static final long AZURE_BLOB_MAX_BLOCK_SIZE = 100L * 1024L * 1024L; - - /** - * Default number of retry attempts (4) for failed requests in Azure SDK 12 - */ - public static final int AZURE_BLOB_MAX_RETRY_REQUESTS = 4; - - /** - * Default timeout duration (60 seconds) for Azure Blob Storage operations - */ - public static final int AZURE_BLOB_DEFAULT_TIMEOUT_SECONDS = 60; - private AzureConstants() { } } diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java index 6020e17f267..a89b99259cb 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java @@ -25,8 +25,8 @@ import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8; -import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier; import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException; @@ -50,18 +50,20 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co private AbstractAzureBlobStoreBackend azureBlobStoreBackend; - private static final String AZURE_SDK_12_ENABLED = "blob.azure.v12.enabled"; - @Override protected AbstractSharedBackend createBackend() { - boolean useAzureSdkV12 = SystemPropertySupplier.create(AZURE_SDK_12_ENABLED, false).get(); - - if (useAzureSdkV12) { - log.info("Starting blob store using Azure SDK 12"); - azureBlobStoreBackend = new AzureBlobStoreBackend(); - } else { - log.info("Starting blob store using Azure SDK 8"); - azureBlobStoreBackend = new AzureBlobStoreBackendV8(); + AzureSdkVersion version = AzureSdkVersion.resolve(properties); + + switch (version) { + case V12: + log.info("Starting blob store using Azure SDK 12"); + azureBlobStoreBackend = new AzureBlobStoreBackendV12(); + break; + case V8: + default: + log.info("Starting blob store using Azure SDK 8"); + azureBlobStoreBackend = new AzureBlobStoreBackendV8(); + break; } if (properties != null) { @@ -138,7 +140,9 @@ public void setDirectDownloadURIExpirySeconds(int seconds) { @Override public void setDirectDownloadURICacheSize(int maxSize) { - azureBlobStoreBackend.setHttpDownloadURICacheSize(maxSize); + if (azureBlobStoreBackend != null) { + azureBlobStoreBackend.setHttpDownloadURICacheSize(maxSize); + } } @Nullable diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureSdkVersion.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureSdkVersion.java new file mode 100644 index 00000000000..bd9a90c995a --- /dev/null +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureSdkVersion.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; + +import java.util.Properties; + +import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * The single point of SDK version selection for Azure blob storage. + *

+ * Resolves the active SDK version from configuration properties (preferred) + * or the {@code blob.azure.v12.enabled} system property (fallback). + */ +public enum AzureSdkVersion { + V8, V12; + + /** + * Resolves the SDK version to use. Checks the given properties first; + * falls back to the system property {@code blob.azure.v12.enabled}. + */ + @NotNull + public static AzureSdkVersion resolve(@Nullable Properties properties) { + if (properties != null) { + String configuredValue = properties.getProperty(AzureConstants.AZURE_V12_ENABLED_PROPERTY); + if (configuredValue != null) { + return Boolean.parseBoolean(configuredValue) ? V12 : V8; + } + } + boolean sysProp = SystemPropertySupplier.create(AzureConstants.AZURE_V12_ENABLED_PROPERTY, false).get(); + return sysProp ? V12 : V8; + } +} diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerProviderV12.java similarity index 81% rename from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java rename to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerProviderV12.java index 25227478113..2e1c9670163 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerProviderV12.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; import com.azure.identity.ClientSecretCredential; import com.azure.identity.ClientSecretCredentialBuilder; @@ -42,8 +42,8 @@ import java.time.ZoneOffset; import java.util.Properties; -public class AzureBlobContainerProvider { - private static final Logger log = LoggerFactory.getLogger(AzureBlobContainerProvider.class); +public class AzureBlobContainerProviderV12 { + private static final Logger log = LoggerFactory.getLogger(AzureBlobContainerProviderV12.class); private static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net"; private final String azureConnectionString; private final String accountName; @@ -55,7 +55,7 @@ public class AzureBlobContainerProvider { private final String clientId; private final String clientSecret; - private AzureBlobContainerProvider(Builder builder) { + private AzureBlobContainerProviderV12(Builder builder) { this.azureConnectionString = builder.azureConnectionString; this.accountName = builder.accountName; this.containerName = builder.containerName; @@ -128,19 +128,19 @@ public Builder withClientSecret(String clientSecret) { } public Builder initializeWithProperties(Properties properties) { - withAzureConnectionString(properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "")); - withAccountName(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "")); - withBlobEndpoint(properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "")); - withSasToken(properties.getProperty(AzureConstants.AZURE_SAS, "")); - withAccountKey(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "")); - withTenantId(properties.getProperty(AzureConstants.AZURE_TENANT_ID, "")); - withClientId(properties.getProperty(AzureConstants.AZURE_CLIENT_ID, "")); - withClientSecret(properties.getProperty(AzureConstants.AZURE_CLIENT_SECRET, "")); + withAzureConnectionString(properties.getProperty(AzureConstantsV12.AZURE_CONNECTION_STRING, "")); + withAccountName(properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, "")); + withBlobEndpoint(properties.getProperty(AzureConstantsV12.AZURE_BLOB_ENDPOINT, "")); + withSasToken(properties.getProperty(AzureConstantsV12.AZURE_SAS, "")); + withAccountKey(properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, "")); + withTenantId(properties.getProperty(AzureConstantsV12.AZURE_TENANT_ID, "")); + withClientId(properties.getProperty(AzureConstantsV12.AZURE_CLIENT_ID, "")); + withClientSecret(properties.getProperty(AzureConstantsV12.AZURE_CLIENT_SECRET, "")); return this; } - public AzureBlobContainerProvider build() { - return new AzureBlobContainerProvider(this); + public AzureBlobContainerProviderV12 build() { + return new AzureBlobContainerProviderV12(this); } } @@ -162,18 +162,18 @@ public BlobContainerClient getBlobContainer(@Nullable RequestRetryOptions retryO // connection string will be given preference over service principals / sas / account key if (StringUtils.isNotBlank(azureConnectionString)) { log.debug("connecting to azure blob storage via azureConnectionString"); - return Utils.getBlobContainerFromConnectionString(getAzureConnectionString(), containerName); + return UtilsV12.getBlobContainerFromConnectionString(getAzureConnectionString(), containerName); } else if (authenticateViaServicePrincipal()) { log.debug("connecting to azure blob storage via service principal credentials"); return getBlobContainerFromServicePrincipals(accountName, retryOptions); } else if (StringUtils.isNotBlank(sasToken)) { log.debug("connecting to azure blob storage via sas token"); - final String connectionStringWithSasToken = Utils.getConnectionStringForSas(sasToken, blobEndpoint, accountName); - return Utils.getBlobContainer(connectionStringWithSasToken, containerName, retryOptions, properties); + final String connectionStringWithSasToken = UtilsV12.getConnectionStringForSas(sasToken, blobEndpoint, accountName); + return UtilsV12.getBlobContainer(connectionStringWithSasToken, containerName, retryOptions, properties); } log.debug("connecting to azure blob storage via access key"); - final String connectionStringWithAccountKey = Utils.getConnectionString(accountName, accountKey, blobEndpoint); - return Utils.getBlobContainer(connectionStringWithAccountKey, containerName, retryOptions, properties); + final String connectionStringWithAccountKey = UtilsV12.getConnectionString(accountName, accountKey, blobEndpoint); + return UtilsV12.getBlobContainer(connectionStringWithAccountKey, containerName, retryOptions, properties); } @NotNull @@ -206,7 +206,7 @@ public String generateSharedAccessSignature(RequestRetryOptions retryOptions, BlobSasPermission blobSasPermissions, int expirySeconds, Properties properties, - @Nullable BlobSasHeaders optionalHeaders) throws DataStoreException, URISyntaxException, InvalidKeyException { + @Nullable BlobSasHeadersV12 optionalHeaders) throws DataStoreException, URISyntaxException, InvalidKeyException { OffsetDateTime expiry = OffsetDateTime.now().plusSeconds(expirySeconds); BlobServiceSasSignatureValues serviceSasSignatureValues = new BlobServiceSasSignatureValues(expiry, blobSasPermissions); @@ -229,14 +229,7 @@ public String generateUserDelegationKeySignedSas(BlockBlobClient blobClient, BlobServiceSasSignatureValues serviceSasSignatureValues, OffsetDateTime expiryTime) { - AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy(); - - String endpoint = getEndpointUrl(accountName, blobEndpoint); - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder() - .endpoint(endpoint) - .credential(getClientSecretCredential()) - .addPolicy(loggingPolicy) - .buildClient(); + BlobServiceClient blobServiceClient = getOrCreateServicePrincipalClient(); OffsetDateTime startTime = OffsetDateTime.now(ZoneOffset.UTC); UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(startTime, expiryTime); return blobClient.generateUserDelegationSas(serviceSasSignatureValues, userDelegationKey); @@ -247,6 +240,28 @@ private boolean authenticateViaServicePrincipal() { StringUtils.isNoneBlank(accountName, tenantId, clientId, clientSecret); } + private volatile BlobServiceClient cachedServicePrincipalClient; + + private BlobServiceClient getOrCreateServicePrincipalClient() { + BlobServiceClient client = cachedServicePrincipalClient; + if (client == null) { + synchronized (this) { + client = cachedServicePrincipalClient; + if (client == null) { + AzureHttpRequestLoggingPolicyV12 loggingPolicy = new AzureHttpRequestLoggingPolicyV12(); + String endpoint = getEndpointUrl(accountName, blobEndpoint); + client = new BlobServiceClientBuilder() + .endpoint(endpoint) + .credential(getClientSecretCredential()) + .addPolicy(loggingPolicy) + .buildClient(); + cachedServicePrincipalClient = client; + } + } + } + return client; + } + private ClientSecretCredential getClientSecretCredential() { return new ClientSecretCredentialBuilder() .clientId(clientId) @@ -258,7 +273,7 @@ private ClientSecretCredential getClientSecretCredential() { @NotNull private BlobContainerClient getBlobContainerFromServicePrincipals(String accountName, RequestRetryOptions retryOptions) { ClientSecretCredential clientSecretCredential = getClientSecretCredential(); - AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 loggingPolicy = new AzureHttpRequestLoggingPolicyV12(); String endpoint = getEndpointUrl(accountName, blobEndpoint); return new BlobContainerClientBuilder() @@ -298,4 +313,4 @@ private static String getEndpointUrl(String accountName, String customBlobEndpoi // Default public endpoint return String.format("https://%s.blob.%s", accountName, DEFAULT_ENDPOINT_SUFFIX); } -} \ No newline at end of file +} diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerV12.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerV12.java new file mode 100644 index 00000000000..a2a08f1e590 --- /dev/null +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerV12.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import java.io.InputStream; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Objects; + +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.sas.BlobContainerSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainer; + +public class AzureBlobContainerV12 implements AzureBlobContainer { + private final BlobContainerClient container; + + public AzureBlobContainerV12(BlobContainerClient container) { + this.container = Objects.requireNonNull(container); + } + + @Override + public void createIfNotExists() { + container.createIfNotExists(); + } + + @Override + public void delete() { + container.delete(); + } + + @Override + public boolean deleteIfExists() { + return container.deleteIfExists(); + } + + @Override + public boolean exists() { + return container.exists(); + } + + @Override + public String getName() { + return container.getBlobContainerName(); + } + + @Override + public String getContainerUri() { + return container.getBlobContainerUrl(); + } + + @Override + public void uploadBlockBlob(String name, InputStream input, long length) { + container.getBlobClient(name).upload(input, length, true); + } + + @Override + public String generateSharedAccessSignature(Instant expiry) { + BlobContainerSasPermission permissions = new BlobContainerSasPermission().setReadPermission(true).setListPermission(true); + BlobServiceSasSignatureValues values = new BlobServiceSasSignatureValues(OffsetDateTime.ofInstant(expiry, ZoneOffset.UTC), permissions); + return container.generateSas(values); + } +} diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobStoreBackendV12.java similarity index 88% rename from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java rename to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobStoreBackendV12.java index 9a53f45ac75..d1205117370 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobStoreBackendV12.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; import com.azure.core.http.rest.Response; import com.azure.storage.blob.BlobClient; @@ -33,7 +33,6 @@ import com.azure.storage.blob.sas.BlobSasPermission; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.common.policy.RequestRetryOptions; -import com.microsoft.azure.storage.RetryPolicy; import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; @@ -41,6 +40,7 @@ import org.apache.jackrabbit.guava.common.cache.Cache; import org.apache.jackrabbit.guava.common.cache.CacheBuilder; import org.apache.jackrabbit.oak.commons.PropertiesUtil; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AbstractAzureBlobStoreBackend; import org.apache.jackrabbit.oak.commons.conditions.Validate; import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord; import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend; @@ -90,26 +90,28 @@ import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadToken; import static java.nio.file.StandardOpenOption.DELETE_ON_CLOSE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_META_KEY_PREFIX; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY; - - -public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend { - - private static final Logger LOG = LoggerFactory.getLogger(AzureBlobStoreBackend.class); +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_LAST_MODIFIED_KEY; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_MAX_BLOCK_SIZE; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_META_KEY_PREFIX; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_REF_KEY; + + +public class AzureBlobStoreBackendV12 extends AbstractAzureBlobStoreBackend { + + private static final Logger LOG = LoggerFactory.getLogger(AzureBlobStoreBackendV12.class); private static final Logger LOG_STREAMS_DOWNLOAD = LoggerFactory.getLogger("oak.datastore.download.streams"); private static final Logger LOG_STREAMS_UPLOAD = LoggerFactory.getLogger("oak.datastore.upload.streams"); private Properties properties; - private AzureBlobContainerProvider azureBlobContainerProvider; + private AzureBlobContainerProviderV12 azureBlobContainerProvider; private int concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; private RequestRetryOptions retryOptions; private Integer requestTimeout; @@ -117,7 +119,7 @@ public class AzureBlobStoreBackend extends AbstractAzureBlobStoreBackend { private int httpUploadURIExpirySeconds = 0; // disabled by default private String uploadDomainOverride = null; private String downloadDomainOverride = null; - private boolean presignedDownloadURIVerifyExists = true; + private boolean presignedDownloadURIVerifyExists = true; private Cache httpDownloadURICache; @@ -129,8 +131,17 @@ public void setProperties(final Properties properties) { private final AtomicReference azureContainerReference = new AtomicReference<>(); protected BlobContainerClient getAzureContainer() throws DataStoreException { - azureContainerReference.compareAndSet(null, azureBlobContainerProvider.getBlobContainer(retryOptions, properties)); - return azureContainerReference.get(); + BlobContainerClient container = azureContainerReference.get(); + if (container == null) { + synchronized (this) { + container = azureContainerReference.get(); + if (container == null) { + container = azureBlobContainerProvider.getBlobContainer(retryOptions, properties); + azureContainerReference.set(container); + } + } + } + return container; } @Override @@ -143,19 +154,19 @@ public void init() throws DataStoreException { if (properties == null) { try { - properties = Utils.readConfig(Utils.DEFAULT_CONFIG_FILE); + properties = UtilsV12.readConfig(UtilsV12.DEFAULT_CONFIG_FILE); } catch (IOException e) { - throw new DataStoreException("Unable to initialize Azure Data Store from " + Utils.DEFAULT_CONFIG_FILE, e); + throw new DataStoreException("Unable to initialize Azure Data Store from " + UtilsV12.DEFAULT_CONFIG_FILE, e); } } try { - boolean createBlobContainer = PropertiesUtil.toBoolean( - org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true); + boolean createBlobContainer = PropertiesUtil.toBoolean( + org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstantsV12.AZURE_CREATE_CONTAINER)), true); initAzureDSConfig(); concurrentRequestCount = PropertiesUtil.toInteger( - properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION), + properties.getProperty(AzureConstantsV12.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION), AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); if (concurrentRequestCount < AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) { LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}", @@ -170,14 +181,14 @@ public void init() throws DataStoreException { } LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount); - if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) { - requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT); + if (properties.getProperty(AzureConstantsV12.AZURE_BLOB_REQUEST_TIMEOUT) != null) { + requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstantsV12.AZURE_BLOB_REQUEST_TIMEOUT), 3); } - retryOptions = Utils.getRetryOptions(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY), requestTimeout, computeSecondaryLocationEndpoint()); + retryOptions = UtilsV12.getRetryOptions(properties.getProperty(AzureConstantsV12.AZURE_BLOB_MAX_REQUEST_RETRY), requestTimeout, computeSecondaryLocationEndpoint()); presignedDownloadURIVerifyExists = PropertiesUtil.toBoolean( - org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true); + org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true); BlobContainerClient azureContainer = getAzureContainer(); @@ -190,26 +201,26 @@ public void init() throws DataStoreException { LOG.debug("Backend initialized. duration={}", stopwatch.elapsed(TimeUnit.MILLISECONDS)); // settings pertaining to DataRecordAccessProvider functionality - String putExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS); + String putExpiry = properties.getProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS); if (putExpiry != null) { this.setHttpUploadURIExpirySeconds(Integer.parseInt(putExpiry)); } - String getExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS); + String getExpiry = properties.getProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS); if (getExpiry != null) { this.setHttpDownloadURIExpirySeconds(Integer.parseInt(getExpiry)); - String cacheMaxSize = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE); + String cacheMaxSize = properties.getProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE); if (cacheMaxSize != null) { this.setHttpDownloadURICacheSize(Integer.parseInt(cacheMaxSize)); } else { this.setHttpDownloadURICacheSize(0); // default } } - uploadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null); - downloadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); + uploadDomainOverride = properties.getProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null); + downloadDomainOverride = properties.getProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); // Initialize reference key secret boolean createRefSecretOnInit = PropertiesUtil.toBoolean( - org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true); + org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstantsV12.AZURE_REF_ON_INIT)), true); if (createRefSecretOnInit) { getOrCreateReferenceKey(); } @@ -223,15 +234,15 @@ public void init() throws DataStoreException { } private void initAzureDSConfig() { - AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder(properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME)) - .withAzureConnectionString(properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "")) - .withAccountName(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "")) - .withBlobEndpoint(properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "")) - .withSasToken(properties.getProperty(AzureConstants.AZURE_SAS, "")) - .withAccountKey(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "")) - .withTenantId(properties.getProperty(AzureConstants.AZURE_TENANT_ID, "")) - .withClientId(properties.getProperty(AzureConstants.AZURE_CLIENT_ID, "")) - .withClientSecret(properties.getProperty(AzureConstants.AZURE_CLIENT_SECRET, "")); + AzureBlobContainerProviderV12.Builder builder = AzureBlobContainerProviderV12.Builder.builder(properties.getProperty(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME)) + .withAzureConnectionString(properties.getProperty(AzureConstantsV12.AZURE_CONNECTION_STRING, "")) + .withAccountName(properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, "")) + .withBlobEndpoint(properties.getProperty(AzureConstantsV12.AZURE_BLOB_ENDPOINT, "")) + .withSasToken(properties.getProperty(AzureConstantsV12.AZURE_SAS, "")) + .withAccountKey(properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, "")) + .withTenantId(properties.getProperty(AzureConstantsV12.AZURE_TENANT_ID, "")) + .withClientId(properties.getProperty(AzureConstantsV12.AZURE_CLIENT_ID, "")) + .withClientSecret(properties.getProperty(AzureConstantsV12.AZURE_CLIENT_SECRET, "")); azureBlobContainerProvider = builder.build(); } @@ -281,7 +292,7 @@ private void tryClose(InputStream is) { private void uploadBlob(BlockBlobClient client, File file, long len, Stopwatch stopwatch, String key) throws IOException { ParallelTransferOptions parallelTransferOptions = new ParallelTransferOptions() - .setBlockSizeLong(len) + .setBlockSizeLong(Math.max(1, Math.min(len, AZURE_BLOB_MAX_BLOCK_SIZE))) .setMaxConcurrency(concurrentRequestCount) .setMaxSingleUploadSizeLong(AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE); BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(file.getPath()); @@ -291,8 +302,8 @@ private void uploadBlob(BlockBlobClient client, File file, long len, Stopwatch s Response blockBlob = blobClient.uploadFromFileWithResponse(options, null, null); LOG.debug("Upload status is {} for blob {}", blockBlob.getStatusCode(), key); } catch (UncheckedIOException ex) { - LOG.debug("Failed to upload from file:{}}", ex.getMessage()); - throw new IOException("Failed to upload blob: " + key, ex); + LOG.debug("Failed to upload from file: {}", ex.getMessage()); + throw ex.getCause(); } LOG.debug("Blob created. identifier={} length={} duration={}", key, len, stopwatch.elapsed(TimeUnit.MILLISECONDS)); if (LOG_STREAMS_UPLOAD.isDebugEnabled()) { @@ -356,10 +367,11 @@ public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - BlockBlobClient blob = getAzureContainer().getBlobClient(key).getBlockBlobClient(); + BlobContainerClient containerClient = getAzureContainer(); + BlockBlobClient blob = containerClient.getBlobClient(key).getBlockBlobClient(); AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord( this, - azureBlobContainerProvider, + containerClient, new DataIdentifier(getIdentifierName(blob.getBlobName())), getLastModified(blob), blob.getProperties().getBlobSize()); @@ -383,6 +395,7 @@ public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException @Override public Iterator getAllIdentifiers() throws DataStoreException { return getAzureContainer().listBlobs().stream() + .filter(blobItem -> blobItem.getName().contains(UtilsV12.DASH)) .map(blobItem -> new DataIdentifier(getIdentifierName(blobItem.getName()))) .iterator(); } @@ -392,9 +405,10 @@ public Iterator getAllRecords() throws DataStoreException { final AbstractSharedBackend backend = this; final BlobContainerClient containerClient = getAzureContainer(); return containerClient.listBlobs().stream() + .filter(blobItem -> blobItem.getName().contains(UtilsV12.DASH)) .map(blobItem -> (DataRecord) new AzureBlobStoreDataRecord( backend, - azureBlobContainerProvider, + containerClient, new DataIdentifier(getIdentifierName(blobItem.getName())), getLastModified(containerClient.getBlobClient(blobItem.getName()).getBlockBlobClient()), blobItem.getProperties().getContentLength())) @@ -492,7 +506,7 @@ public void addMetadataRecord(File inputFile, String name) throws DataStoreExcep } private BlockBlobClient getMetaBlobClient(String name) throws DataStoreException { - return getAzureContainer().getBlobClient(AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + name).getBlockBlobClient(); + return getAzureContainer().getBlobClient(AzureConstantsV12.AZURE_BLOB_META_DIR_NAME + "/" + name).getBlockBlobClient(); } private void addMetadataRecordImpl(final InputStream input, String name, long recordLength) throws DataStoreException { @@ -572,6 +586,9 @@ private Path createTempFileFromStream(InputStream input, String name) throws IOE @Override public DataRecord getMetadataRecord(String name) { + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException("name must not be null or empty"); + } ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); Stopwatch stopwatch = Stopwatch.createStarted(); try { @@ -586,7 +603,7 @@ public DataRecord getMetadataRecord(String name) { long lastModified = getLastModified(blockBlobClient); long length = blockBlobClient.getProperties().getBlobSize(); AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(this, - azureBlobContainerProvider, + getAzureContainer(), new DataIdentifier(name), lastModified, length, @@ -614,14 +631,15 @@ public List getAllMetadataRecords(String prefix) { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); ListBlobsOptions listBlobsOptions = new ListBlobsOptions(); - listBlobsOptions.setPrefix(AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + prefix); + listBlobsOptions.setPrefix(AzureConstantsV12.AZURE_BLOB_META_DIR_NAME + "/" + prefix); - for (BlobItem blobItem : getAzureContainer().listBlobs(listBlobsOptions, null)) { - BlobClient blobClient = getAzureContainer().getBlobClient(blobItem.getName()); + BlobContainerClient containerClient = getAzureContainer(); + for (BlobItem blobItem : containerClient.listBlobs(listBlobsOptions, null)) { + BlobClient blobClient = containerClient.getBlobClient(blobItem.getName()); BlobProperties properties = blobClient.getProperties(); records.add(new AzureBlobStoreDataRecord(this, - azureBlobContainerProvider, + containerClient, new DataIdentifier(stripMetaKeyPrefix(blobClient.getBlobName())), getLastModified(blobClient.getBlockBlobClient()), properties.getBlobSize(), @@ -673,10 +691,11 @@ public void deleteAllMetadataRecords(String prefix) { int total = 0; ListBlobsOptions listBlobsOptions = new ListBlobsOptions(); - listBlobsOptions.setPrefix(AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + prefix); + listBlobsOptions.setPrefix(AzureConstantsV12.AZURE_BLOB_META_DIR_NAME + "/" + prefix); - for (BlobItem blobItem : getAzureContainer().listBlobs(listBlobsOptions, null)) { - BlobClient blobClient = getAzureContainer().getBlobClient(blobItem.getName()); + BlobContainerClient containerClient = getAzureContainer(); + for (BlobItem blobItem : containerClient.listBlobs(listBlobsOptions, null)) { + BlobClient blobClient = containerClient.getBlobClient(blobItem.getName()); if (blobClient.deleteIfExists()) { total++; } @@ -718,14 +737,14 @@ public boolean metadataRecordExists(String name) { */ private static String getKeyName(DataIdentifier identifier) { String key = identifier.toString(); - return key.substring(0, 4) + Utils.DASH + key.substring(4); + return key.substring(0, 4) + UtilsV12.DASH + key.substring(4); } /** * Get data identifier from key. */ private static String getIdentifierName(String key) { - if (!key.contains(Utils.DASH)) { + if (!key.contains(UtilsV12.DASH)) { return null; } else if (key.contains(AZURE_BLOB_META_KEY_PREFIX)) { return key; @@ -758,11 +777,11 @@ private static long getLastModified(BlockBlobClient blockBlobClient) { return Long.parseLong(metadata.get(AZURE_BLOB_LAST_MODIFIED_KEY)); } - protected void setHttpDownloadURIExpirySeconds(int seconds) { + public void setHttpDownloadURIExpirySeconds(int seconds) { httpDownloadURIExpirySeconds = seconds; } - protected void setHttpDownloadURICacheSize(int maxSize) { + public void setHttpDownloadURICacheSize(int maxSize) { // max size 0 or smaller is used to turn off the cache if (maxSize > 0) { LOG.info("presigned GET URI cache enabled, maxSize = {} items, expiry = {} seconds", maxSize, httpDownloadURIExpirySeconds / 2); @@ -777,7 +796,7 @@ protected void setHttpDownloadURICacheSize(int maxSize) { } @Override - protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier, + public URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions) { URI uri = null; @@ -814,7 +833,7 @@ protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier, String key = getKeyName(identifier); // Prepare headers for the presigned URI - BlobSasHeaders headers = new BlobSasHeaders() + BlobSasHeadersV12 headers = new BlobSasHeadersV12() .setCacheControl(String.format("private, max-age=%d, immutable", httpDownloadURIExpirySeconds)) .setContentType(downloadOptions.getContentTypeHeader()) .setContentDisposition(downloadOptions.getContentDispositionHeader()); @@ -833,7 +852,7 @@ protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier, return uri; } - protected void setHttpUploadURIExpirySeconds(int seconds) { + public void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; } @@ -846,7 +865,7 @@ private DataIdentifier generateSafeRandomIdentifier() { ); } - protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) { + public DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) { List uploadPartURIs = new ArrayList<>(); long minPartSize = AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE; long maxPartSize = AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; @@ -966,7 +985,7 @@ private Long getUncommittedBlocksListSize(BlockBlobClient client) throws DataSto return size; } - protected DataRecord completeHttpUpload(@NotNull String uploadTokenStr) + public DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException { Validate.checkArgument(StringUtils.isNotEmpty(uploadTokenStr), "uploadToken required"); @@ -989,7 +1008,7 @@ record = getRecord(blobId); long size = getUncommittedBlocksListSize(blockBlobClient); record = new AzureBlobStoreDataRecord( this, - azureBlobContainerProvider, + getAzureContainer(), blobId, getLastModified(blockBlobClient), size); @@ -1013,7 +1032,7 @@ record = new AzureBlobStoreDataRecord( } private String getDefaultBlobStorageDomain() { - String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); + String accountName = properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, ""); if (StringUtils.isEmpty(accountName)) { LOG.warn("Can't generate presigned URI - Azure account name not found in properties"); return null; @@ -1061,7 +1080,7 @@ private URI createPresignedURI(String key, int expirySeconds, Map additionalQueryParams, String domain, - BlobSasHeaders optionalHeaders) { + BlobSasHeadersV12 optionalHeaders) { if (Objects.toString(domain, "").isEmpty()) { LOG.warn("Can't generate presigned URI - no Azure domain provided (is Azure account name configured?)"); return null; @@ -1110,20 +1129,22 @@ private URI createPresignedURI(String key, } static class AzureBlobStoreDataRecord extends AbstractDataRecord { - final AzureBlobContainerProvider azureBlobContainerProvider; + final BlobContainerClient blobContainerClient; + final String containerName; final long lastModified; final long length; final boolean isMeta; - public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProvider azureBlobContainerProvider, - DataIdentifier key, long lastModified, long length) { - this(backend, azureBlobContainerProvider, key, lastModified, length, false); + public AzureBlobStoreDataRecord(AbstractSharedBackend backend, + BlobContainerClient blobContainerClient, DataIdentifier key, long lastModified, long length) { + this(backend, blobContainerClient, key, lastModified, length, false); } - public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProvider azureBlobContainerProvider, - DataIdentifier key, long lastModified, long length, boolean isMeta) { + public AzureBlobStoreDataRecord(AbstractSharedBackend backend, + BlobContainerClient blobContainerClient, DataIdentifier key, long lastModified, long length, boolean isMeta) { super(backend, key); - this.azureBlobContainerProvider = azureBlobContainerProvider; + this.blobContainerClient = blobContainerClient; + this.containerName = blobContainerClient != null ? blobContainerClient.getBlobContainerName() : null; this.lastModified = lastModified; this.length = length; this.isMeta = isMeta; @@ -1137,7 +1158,6 @@ public long getLength() throws DataStoreException { @Override public InputStream getStream() throws DataStoreException { String id = getKeyName(getIdentifier()); - BlobContainerClient container = azureBlobContainerProvider.getBlobContainer(); if (isMeta) { id = addMetaKeyPrefix(getIdentifier().toString()); } @@ -1149,7 +1169,7 @@ public InputStream getStream() throws DataStoreException { } } try { - return container.getBlobClient(id).openInputStream(); + return blobContainerClient.getBlobClient(id).openInputStream(); } catch (Exception e) { throw new DataStoreException(e); } @@ -1166,14 +1186,14 @@ public String toString() { "identifier=" + getIdentifier() + ", length=" + length + ", lastModified=" + lastModified + - ", containerName='" + Optional.ofNullable(azureBlobContainerProvider).map(AzureBlobContainerProvider::getContainerName).orElse(null) + '\'' + + ", containerName='" + containerName + '\'' + '}'; } } private String getContainerName() { return Optional.ofNullable(this.azureBlobContainerProvider) - .map(AzureBlobContainerProvider::getContainerName) + .map(AzureBlobContainerProviderV12::getContainerName) .orElse(null); } @@ -1215,10 +1235,10 @@ protected byte[] readMetadataBytes(String name) throws IOException, DataStoreExc } private String computeSecondaryLocationEndpoint() { - String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); + String accountName = properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, ""); - boolean enableSecondaryLocation = PropertiesUtil.toBoolean(properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME), - AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT); + boolean enableSecondaryLocation = PropertiesUtil.toBoolean(properties.getProperty(AzureConstantsV12.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME), + AzureConstantsV12.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT); if(enableSecondaryLocation) { return String.format("https://%s-secondary.blob.core.windows.net", accountName); diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureConstantsV12.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureConstantsV12.java new file mode 100644 index 00000000000..9759d9dd3f2 --- /dev/null +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureConstantsV12.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +/** + * Constants for the Azure Blob Storage v12 SDK backend. + *

+ * These values are self-contained within the v12 package. They must not reference + * or delegate to the shared {@code AzureConstants} class, so that the v12 package + * remains fully independent of other packages. + */ +public final class AzureConstantsV12 { + + // --- Configuration property names --- + + public static final String AZURE_STORAGE_ACCOUNT_NAME = "accessKey"; + public static final String AZURE_STORAGE_ACCOUNT_KEY = "secretKey"; + public static final String AZURE_CONNECTION_STRING = "azureConnectionString"; + public static final String AZURE_SAS = "azureSas"; + public static final String AZURE_TENANT_ID = "tenantId"; + public static final String AZURE_CLIENT_ID = "clientId"; + public static final String AZURE_CLIENT_SECRET = "clientSecret"; + public static final String AZURE_BLOB_ENDPOINT = "azureBlobEndpoint"; + public static final String AZURE_BLOB_CONTAINER_NAME = "container"; + public static final String AZURE_CREATE_CONTAINER = "azureCreateContainer"; + public static final String AZURE_BLOB_REQUEST_TIMEOUT = "socketTimeout"; + public static final String AZURE_BLOB_MAX_REQUEST_RETRY = "maxErrorRetry"; + public static final String AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION = "maxConnections"; + public static final String AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME = "enableSecondaryLocation"; + public static final boolean AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT = false; + public static final String PROXY_HOST = "proxyHost"; + public static final String PROXY_PORT = "proxyPort"; + public static final String PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS = "presignedHttpUploadURIExpirySeconds"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS = "presignedHttpDownloadURIExpirySeconds"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE = "presignedHttpDownloadURICacheMaxSize"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS = "presignedHttpDownloadURIVerifyExists"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpDownloadURIDomainOverride"; + public static final String PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpUploadURIDomainOverride"; + public static final String AZURE_REF_ON_INIT = "refOnInit"; + + // --- v12-specific runtime constants --- + // These values differ from V8 to leverage Azure SDK v12 capabilities: + // - Higher concurrency default (5 vs V8's 2) with lower max (10 vs V8's 50) for better default throughput + // - Larger buffer threshold (8 MB vs V8's 1 MB) to reduce small-buffer overhead + // - Smaller min part size (4 MB vs V8's 10 MB) matching SDK v12's minimum block size + // - Max upload size of 190 TiB (50,000 blocks x 4000 MiB) vs V8's ~4.75 TiB limit + + public static final String AZURE_BLOB_META_DIR_NAME = "META"; + public static final String AZURE_BLOB_META_KEY_PREFIX = AZURE_BLOB_META_DIR_NAME + "/"; + public static final String AZURE_BLOB_REF_KEY = "reference.key"; + public static final String AZURE_BLOB_LAST_MODIFIED_KEY = "lastModified"; + public static final long AZURE_BLOB_BUFFERED_STREAM_THRESHOLD = 8L * 1024L * 1024L; // 8 MiB + public static final long AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE = 4L * 1024L * 1024L; // 4 MiB (SDK v12 minimum block size) + public static final long AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE = 4000L * 1024L * 1024L; // 4000 MiB (SDK v12 maximum block size) + public static final long AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE = 256L * 1024L * 1024L; // 256 MiB + public static final long AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE = 190L * 1024L * 1024L * 1024L * 1024L; // ~190 TiB (50,000 blocks x 4000 MiB) + public static final int AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS = 50000; + public static final int AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT = 5; + public static final int AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT = 10; + public static final long AZURE_BLOB_MAX_BLOCK_SIZE = 100L * 1024L * 1024L; // 100 MiB + public static final int AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES = 10; + + private AzureConstantsV12() { + } +} \ No newline at end of file diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicy.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureHttpRequestLoggingPolicyV12.java similarity index 88% rename from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicy.java rename to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureHttpRequestLoggingPolicyV12.java index 70f18568ed9..296bd98f251 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicy.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureHttpRequestLoggingPolicyV12.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; @@ -36,14 +36,14 @@ * * This policy logs HTTP request details including method, URL, status code, and duration. * Verbose logging can be enabled by setting the system property: - * -Dblob.azure.http.verbose.enabled=true + * -Dblob.azure.v12.http.verbose.enabled=true * - * This is similar to the AzureHttpRequestLoggingPolicy in oak-segment-azure but specifically + * This is similar to the AzureHttpRequestLoggingPolicyV12 in oak-segment-azure but specifically * designed for the blob storage operations in oak-blob-cloud-azure. */ -public class AzureHttpRequestLoggingPolicy implements HttpPipelinePolicy { +public class AzureHttpRequestLoggingPolicyV12 implements HttpPipelinePolicy { - private static final Logger log = LoggerFactory.getLogger(AzureHttpRequestLoggingPolicy.class); + private static final Logger log = LoggerFactory.getLogger(AzureHttpRequestLoggingPolicyV12.class); private static final String AZURE_SDK_VERBOSE_LOGGING_ENABLED = "blob.azure.v12.http.verbose.enabled"; diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/BlobSasHeaders.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/BlobSasHeadersV12.java similarity index 83% rename from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/BlobSasHeaders.java rename to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/BlobSasHeadersV12.java index 1d066eabf6b..3887db5ef69 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/BlobSasHeaders.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/BlobSasHeadersV12.java @@ -16,21 +16,21 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; import org.jetbrains.annotations.Nullable; /** * Represents the optional headers that can be returned using SAS (Shared Access Signature). - * This class is the Azure SDK 12 equivalent of the legacy {@code com.microsoft.azure.storage.blob.SharedAccessBlobHeaders}. + * This class is the Azure SDK 12 equivalent of the legacy V8 {@code SharedAccessBlobHeaders}. * *

These headers are set on the {@link BlobServiceSasSignatureValues} object and will be * returned to the client when the SAS token is used to access the blob.

* * @see BlobServiceSasSignatureValues */ -public class BlobSasHeaders { +public class BlobSasHeadersV12 { private String cacheControl; private String contentDisposition; @@ -39,13 +39,13 @@ public class BlobSasHeaders { private String contentType; /** - * Creates an empty BlobSasHeaders object. + * Creates an empty BlobSasHeadersV12 object. */ - public BlobSasHeaders() { + public BlobSasHeadersV12() { } /** - * Creates a BlobSasHeaders object with the specified values. + * Creates a BlobSasHeadersV12 object with the specified values. * * @param cacheControl the cache-control header value * @param contentDisposition the content-disposition header value @@ -53,7 +53,7 @@ public BlobSasHeaders() { * @param contentLanguage the content-language header value * @param contentType the content-type header value */ - public BlobSasHeaders(@Nullable String cacheControl, + public BlobSasHeadersV12(@Nullable String cacheControl, @Nullable String contentDisposition, @Nullable String contentEncoding, @Nullable String contentLanguage, @@ -79,9 +79,9 @@ public String getCacheControl() { * Sets the cache-control header value. * * @param cacheControl the cache-control header value - * @return this BlobSasHeaders object for method chaining + * @return this BlobSasHeadersV12 object for method chaining */ - public BlobSasHeaders setCacheControl(@Nullable String cacheControl) { + public BlobSasHeadersV12 setCacheControl(@Nullable String cacheControl) { this.cacheControl = cacheControl; return this; } @@ -100,9 +100,9 @@ public String getContentDisposition() { * Sets the content-disposition header value. * * @param contentDisposition the content-disposition header value - * @return this BlobSasHeaders object for method chaining + * @return this BlobSasHeadersV12 object for method chaining */ - public BlobSasHeaders setContentDisposition(@Nullable String contentDisposition) { + public BlobSasHeadersV12 setContentDisposition(@Nullable String contentDisposition) { this.contentDisposition = contentDisposition; return this; } @@ -121,9 +121,9 @@ public String getContentEncoding() { * Sets the content-encoding header value. * * @param contentEncoding the content-encoding header value - * @return this BlobSasHeaders object for method chaining + * @return this BlobSasHeadersV12 object for method chaining */ - public BlobSasHeaders setContentEncoding(@Nullable String contentEncoding) { + public BlobSasHeadersV12 setContentEncoding(@Nullable String contentEncoding) { this.contentEncoding = contentEncoding; return this; } @@ -142,9 +142,9 @@ public String getContentLanguage() { * Sets the content-language header value. * * @param contentLanguage the content-language header value - * @return this BlobSasHeaders object for method chaining + * @return this BlobSasHeadersV12 object for method chaining */ - public BlobSasHeaders setContentLanguage(@Nullable String contentLanguage) { + public BlobSasHeadersV12 setContentLanguage(@Nullable String contentLanguage) { this.contentLanguage = contentLanguage; return this; } @@ -163,9 +163,9 @@ public String getContentType() { * Sets the content-type header value. * * @param contentType the content-type header value - * @return this BlobSasHeaders object for method chaining + * @return this BlobSasHeadersV12 object for method chaining */ - public BlobSasHeaders setContentType(@Nullable String contentType) { + public BlobSasHeadersV12 setContentType(@Nullable String contentType) { this.contentType = contentType; return this; } diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/UtilsV12.java similarity index 77% rename from oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java rename to oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/UtilsV12.java index 1aa1edcacda..30ba88489d7 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/UtilsV12.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; import java.io.File; import java.io.FileInputStream; @@ -41,29 +41,31 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -public final class Utils { +public final class UtilsV12 { public static final String DASH = "-"; public static final String DEFAULT_CONFIG_FILE = "azure.properties"; - private Utils() {} + private UtilsV12() {} public static BlobContainerClient getBlobContainer(@NotNull final String connectionString, @NotNull final String containerName, @Nullable final RequestRetryOptions retryOptions, final Properties properties) throws DataStoreException { try { - AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 loggingPolicy = new AzureHttpRequestLoggingPolicyV12(); BlobServiceClientBuilder builder = new BlobServiceClientBuilder() .connectionString(connectionString) .retryOptions(retryOptions) .addPolicy(loggingPolicy); + ProxyOptions proxyOptions = computeProxyOptions(properties); + if (proxyOptions != null) { HttpClient httpClient = new NettyAsyncHttpClientBuilder() - .proxy(computeProxyOptions(properties)) + .proxy(proxyOptions) .build(); - builder.httpClient(httpClient); + } BlobServiceClient blobServiceClient = builder.buildClient(); return blobServiceClient.getBlobContainerClient(containerName); @@ -74,8 +76,8 @@ public static BlobContainerClient getBlobContainer(@NotNull final String connect } public static ProxyOptions computeProxyOptions(final Properties properties) { - String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST); - String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT); + String proxyHost = properties.getProperty(AzureConstantsV12.PROXY_HOST); + String proxyPort = properties.getProperty(AzureConstantsV12.PROXY_PORT); if(!(Objects.toString(proxyHost, "").isEmpty() || Objects.toString(proxyPort, "").isEmpty())) { return new ProxyOptions(ProxyOptions.Type.HTTP, @@ -101,11 +103,11 @@ public static RequestRetryOptions getRetryOptions(final String maxRequestRetryCo } public static String getConnectionStringFromProperties(Properties properties) { - String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, ""); - String blobEndpoint = properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, ""); - String connectionString = properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, ""); - String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); - String accountKey = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, ""); + String sasUri = properties.getProperty(AzureConstantsV12.AZURE_SAS, ""); + String blobEndpoint = properties.getProperty(AzureConstantsV12.AZURE_BLOB_ENDPOINT, ""); + String connectionString = properties.getProperty(AzureConstantsV12.AZURE_CONNECTION_STRING, ""); + String accountName = properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, ""); + String accountKey = properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, ""); if (!connectionString.isEmpty()) { return connectionString; @@ -140,7 +142,7 @@ public static String getConnectionString(final String accountName, final String } public static BlobContainerClient getBlobContainerFromConnectionString(final String azureConnectionString, final String containerName) { - AzureHttpRequestLoggingPolicy loggingPolicy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 loggingPolicy = new AzureHttpRequestLoggingPolicyV12(); return new BlobContainerClientBuilder() .connectionString(azureConnectionString) @@ -149,6 +151,31 @@ public static BlobContainerClient getBlobContainerFromConnectionString(final Str .buildClient(); } + /** + * Check whether the given properties contain sufficient Azure configuration + * for V12 SDK connectivity (account key, SAS, or AAD credentials). + */ + public static boolean isConfigured(Properties props) { + // Account key auth + if (props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY) + && props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME) + && props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME)) { + return true; + } + // SAS auth + if (props.containsKey(AzureConstantsV12.AZURE_SAS) + && props.containsKey(AzureConstantsV12.AZURE_BLOB_ENDPOINT) + && props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME)) { + return true; + } + // AAD client credentials + return props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME) + && props.containsKey(AzureConstantsV12.AZURE_TENANT_ID) + && props.containsKey(AzureConstantsV12.AZURE_CLIENT_ID) + && props.containsKey(AzureConstantsV12.AZURE_CLIENT_SECRET) + && props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME); + } + /** * Read a configuration properties file. * diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java index b2fbc67ff66..182a46ea0c9 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8.java @@ -35,7 +35,6 @@ import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy; import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -90,6 +89,7 @@ private AzureBlobContainerProviderV8(Builder builder) { this.clientSecret = builder.clientSecret; } + public static class Builder { private final String containerName; @@ -151,14 +151,14 @@ public Builder withClientSecret(String clientSecret) { } public Builder initializeWithProperties(Properties properties) { - withAzureConnectionString(properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "")); - withAccountName(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "")); - withBlobEndpoint(properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "")); - withSasToken(properties.getProperty(AzureConstants.AZURE_SAS, "")); - withAccountKey(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "")); - withTenantId(properties.getProperty(AzureConstants.AZURE_TENANT_ID, "")); - withClientId(properties.getProperty(AzureConstants.AZURE_CLIENT_ID, "")); - withClientSecret(properties.getProperty(AzureConstants.AZURE_CLIENT_SECRET, "")); + withAzureConnectionString(properties.getProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, "")); + withAccountName(properties.getProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "")); + withBlobEndpoint(properties.getProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "")); + withSasToken(properties.getProperty(AzureConstantsV8.AZURE_SAS, "")); + withAccountKey(properties.getProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "")); + withTenantId(properties.getProperty(AzureConstantsV8.AZURE_TENANT_ID, "")); + withClientId(properties.getProperty(AzureConstantsV8.AZURE_CLIENT_ID, "")); + withClientSecret(properties.getProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, "")); return this; } diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerV8.java new file mode 100644 index 00000000000..a1c25eabaab --- /dev/null +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerV8.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8; + +import java.io.InputStream; +import java.time.Instant; +import java.util.EnumSet; +import java.util.Objects; + +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions; +import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainer; + +public class AzureBlobContainerV8 implements AzureBlobContainer { + private final CloudBlobContainer container; + private final AzureBlobContainerProviderV8 provider; + + public AzureBlobContainerV8(CloudBlobContainer container, AzureBlobContainerProviderV8 provider) { + this.container = Objects.requireNonNull(container); + this.provider = Objects.requireNonNull(provider); + } + + @Override + public void createIfNotExists() throws Exception { + container.createIfNotExists(); + } + + @Override + public void delete() throws Exception { + container.delete(); + } + + @Override + public boolean deleteIfExists() throws Exception { + return container.deleteIfExists(); + } + + @Override + public boolean exists() throws Exception { + return container.exists(); + } + + @Override + public String getName() { + return container.getName(); + } + + @Override + public String getContainerUri() { + return container.getUri().toString(); + } + + @Override + public void uploadBlockBlob(String name, InputStream input, long length) throws Exception { + container.getBlockBlobReference(name).upload(input, length); + } + + @Override + public String generateSharedAccessSignature(Instant expiry) throws Exception { + SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy(); + policy.setSharedAccessExpiryTime(java.util.Date.from(expiry)); + policy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ, SharedAccessBlobPermissions.LIST)); + return container.generateSharedAccessSignature(policy, null); + } + + @Override + public void close() throws Exception { + provider.close(); + } +} diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java index 95acd908dd9..63e250cff67 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java @@ -19,18 +19,6 @@ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8; import static java.lang.Thread.currentThread; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BlOB_META_DIR_NAME; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_META_KEY_PREFIX; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -59,14 +47,13 @@ import java.util.Queue; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; -import org.apache.jackrabbit.guava.common.cache.Cache; -import org.apache.jackrabbit.guava.common.cache.CacheBuilder; +import org.apache.commons.lang3.StringUtils; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AbstractAzureBlobStoreBackend; import org.apache.jackrabbit.oak.commons.collections.AbstractIterator; -import org.apache.jackrabbit.oak.commons.time.Stopwatch; - import com.microsoft.azure.storage.AccessCondition; import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.ResultContinuation; @@ -81,19 +68,15 @@ import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.CopyStatus; import com.microsoft.azure.storage.blob.ListBlobItem; import com.microsoft.azure.storage.blob.SharedAccessBlobHeaders; import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AbstractAzureBlobStoreBackend; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils; import org.apache.jackrabbit.oak.commons.PropertiesUtil; -import org.apache.jackrabbit.oak.commons.conditions.Validate; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException; @@ -114,7 +97,7 @@ public class AzureBlobStoreBackendV8 extends AbstractAzureBlobStoreBackend { private Properties properties; private AzureBlobContainerProviderV8 azureBlobContainerProvider; - private int concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; + private int concurrentRequestCount = AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; private RetryPolicy retryPolicy; private Integer requestTimeout; private int httpDownloadURIExpirySeconds = 0; // disabled by default @@ -123,7 +106,7 @@ public class AzureBlobStoreBackendV8 extends AbstractAzureBlobStoreBackend { private String downloadDomainOverride = null; private boolean createBlobContainer = true; private boolean presignedDownloadURIVerifyExists = true; - private boolean enableSecondaryLocation = AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT; + private boolean enableSecondaryLocation = AzureConstantsV8.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT; private Cache httpDownloadURICache; @@ -133,20 +116,26 @@ public void setProperties(final Properties properties) { this.properties = properties; } - private final AtomicReference azureContainerReference = new AtomicReference<>(); + private volatile CloudBlobContainer azureContainer = null; - public CloudBlobContainer getAzureContainer() throws DataStoreException { - azureContainerReference.compareAndSet(null, azureBlobContainerProvider.getBlobContainer(getBlobRequestOptions())); - return azureContainerReference.get(); + protected CloudBlobContainer getAzureContainer() throws DataStoreException { + if (azureContainer == null) { + synchronized (this) { + if (azureContainer == null) { + azureContainer = azureBlobContainerProvider.getBlobContainer(getBlobRequestOptions()); + } + } + } + return azureContainer; } @NotNull protected BlobRequestOptions getBlobRequestOptions() { BlobRequestOptions requestOptions = new BlobRequestOptions(); - if (retryPolicy != null) { + if (null != retryPolicy) { requestOptions.setRetryPolicyFactory(retryPolicy); } - if (requestTimeout != null) { + if (null != requestTimeout) { requestOptions.setTimeoutIntervalInMs(requestTimeout); } requestOptions.setConcurrentRequestCount(concurrentRequestCount); @@ -159,14 +148,14 @@ protected BlobRequestOptions getBlobRequestOptions() { @Override public void init() throws DataStoreException { ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); LOG.debug("Started backend initialization"); - if (properties == null) { + if (null == properties) { try { - properties = Utils.readConfig(UtilsV8.DEFAULT_CONFIG_FILE); + properties = UtilsV8.readConfig(UtilsV8.DEFAULT_CONFIG_FILE); } catch (IOException e) { throw new DataStoreException("Unable to initialize Azure Data Store from " + UtilsV8.DEFAULT_CONFIG_FILE, e); @@ -176,35 +165,35 @@ public void init() throws DataStoreException { try { UtilsV8.setProxyIfNeeded(properties); createBlobContainer = PropertiesUtil.toBoolean( - org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true); + org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstantsV8.AZURE_CREATE_CONTAINER)), true); initAzureDSConfig(); concurrentRequestCount = PropertiesUtil.toInteger( - properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION), - AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); - if (concurrentRequestCount < AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) { + properties.getProperty(AzureConstantsV8.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION), + AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); + if (concurrentRequestCount < AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) { LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}", concurrentRequestCount, - AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); - concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; - } else if (concurrentRequestCount > AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT) { + AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); + concurrentRequestCount = AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; + } else if (concurrentRequestCount > AzureConstantsV8.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT) { LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too high); resetting to {}", concurrentRequestCount, - AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT); - concurrentRequestCount = AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT; + AzureConstantsV8.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT); + concurrentRequestCount = AzureConstantsV8.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT; } LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount); - retryPolicy = UtilsV8.getRetryPolicy(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY)); - if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) { - requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT); + retryPolicy = UtilsV8.getRetryPolicy(properties.getProperty(AzureConstantsV8.AZURE_BLOB_MAX_REQUEST_RETRY)); + if (properties.getProperty(AzureConstantsV8.AZURE_BLOB_REQUEST_TIMEOUT) != null) { + requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstantsV8.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT); } presignedDownloadURIVerifyExists = PropertiesUtil.toBoolean( - org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true); + org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true); enableSecondaryLocation = PropertiesUtil.toBoolean( - properties.getProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME), - AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT + properties.getProperty(AzureConstantsV8.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME), + AzureConstantsV8.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT ); CloudBlobContainer azureContainer = getAzureContainer(); @@ -215,38 +204,37 @@ public void init() throws DataStoreException { } else { LOG.info("Reusing existing container. containerName={}", getContainerName()); } - LOG.debug("Backend initialized. duration={}", watch.elapsed(TimeUnit.MILLISECONDS)); + LOG.debug("Backend initialized. duration={}", (System.currentTimeMillis() - start)); // settings pertaining to DataRecordAccessProvider functionality - String putExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS); - if (putExpiry != null) { + String putExpiry = properties.getProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS); + if (null != putExpiry) { this.setHttpUploadURIExpirySeconds(Integer.parseInt(putExpiry)); } - String getExpiry = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS); - if (getExpiry != null) { + String getExpiry = properties.getProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS); + if (null != getExpiry) { this.setHttpDownloadURIExpirySeconds(Integer.parseInt(getExpiry)); - String cacheMaxSize = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE); - if (cacheMaxSize != null) { + String cacheMaxSize = properties.getProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE); + if (null != cacheMaxSize) { this.setHttpDownloadURICacheSize(Integer.parseInt(cacheMaxSize)); } else { this.setHttpDownloadURICacheSize(0); // default } } - uploadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null); - downloadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); + uploadDomainOverride = properties.getProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null); + downloadDomainOverride = properties.getProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); // Initialize reference key secret boolean createRefSecretOnInit = PropertiesUtil.toBoolean( - org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true); + org.apache.jackrabbit.oak.commons.StringUtils.emptyToNull(properties.getProperty(AzureConstantsV8.AZURE_REF_ON_INIT)), true); if (createRefSecretOnInit) { getOrCreateReferenceKey(); } } catch (StorageException e) { - LOG.error("Error performing blob storage creation: {}", e.getMessage()); - throw new DataStoreException("Error performing backend initialization", e); + throw new DataStoreException(e); } } finally { @@ -255,24 +243,24 @@ public void init() throws DataStoreException { } private void initAzureDSConfig() { - AzureBlobContainerProviderV8.Builder builder = AzureBlobContainerProviderV8.Builder.builder(properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME)) - .withAzureConnectionString(properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "")) - .withAccountName(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "")) - .withBlobEndpoint(properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "")) - .withSasToken(properties.getProperty(AzureConstants.AZURE_SAS, "")) - .withAccountKey(properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "")) - .withTenantId(properties.getProperty(AzureConstants.AZURE_TENANT_ID, "")) - .withClientId(properties.getProperty(AzureConstants.AZURE_CLIENT_ID, "")) - .withClientSecret(properties.getProperty(AzureConstants.AZURE_CLIENT_SECRET, "")); + AzureBlobContainerProviderV8.Builder builder = AzureBlobContainerProviderV8.Builder.builder(properties.getProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME)) + .withAzureConnectionString(properties.getProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, "")) + .withAccountName(properties.getProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "")) + .withBlobEndpoint(properties.getProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "")) + .withSasToken(properties.getProperty(AzureConstantsV8.AZURE_SAS, "")) + .withAccountKey(properties.getProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "")) + .withTenantId(properties.getProperty(AzureConstantsV8.AZURE_TENANT_ID, "")) + .withClientId(properties.getProperty(AzureConstantsV8.AZURE_CLIENT_ID, "")) + .withClientSecret(properties.getProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, "")); azureBlobContainerProvider = builder.build(); } @Override public InputStream read(DataIdentifier identifier) throws DataStoreException { - Objects.requireNonNull(identifier, "identifier must not be null"); + if (null == identifier) throw new NullPointerException("identifier"); String key = getKeyName(identifier); - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( @@ -283,18 +271,21 @@ public InputStream read(DataIdentifier identifier) throws DataStoreException { } InputStream is = blob.openInputStream(); - LOG.debug("Got input stream for blob. identifier={} duration={}", key, watch.elapsed(TimeUnit.MILLISECONDS)); + LOG.debug("Got input stream for blob. identifier={} duration={}", key, (System.currentTimeMillis() - start)); if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) { // Log message, with exception so we can get a trace to see where the call came from LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob Storage - identifier={}", key, new Exception()); } return is; } - catch (StorageException | URISyntaxException e) { - LOG.error("Error reading blob. identifier={}", key); + catch (StorageException e) { + LOG.info("Error reading blob. identifier={}", key); throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e); } - finally { + catch (URISyntaxException e) { + LOG.debug("Error reading blob. identifier={}", key); + throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e); + } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } @@ -303,11 +294,14 @@ public InputStream read(DataIdentifier identifier) throws DataStoreException { @Override public void write(DataIdentifier identifier, File file) throws DataStoreException { - Objects.requireNonNull(identifier, "identifier must not be null"); - Objects.requireNonNull(file, "file must not be null"); - + if (null == identifier) { + throw new NullPointerException("identifier"); + } + if (null == file) { + throw new NullPointerException("file"); + } String key = getKeyName(identifier); - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { @@ -321,23 +315,26 @@ public void write(DataIdentifier identifier, File file) throws DataStoreExceptio BlobRequestOptions options = new BlobRequestOptions(); options.setConcurrentRequestCount(concurrentRequestCount); - boolean useBufferedStream = len < AZURE_BLOB_BUFFERED_STREAM_THRESHOLD; - try (InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file)) { - blob.upload(in, len, null, options, null); - LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, watch.elapsed(TimeUnit.MILLISECONDS), useBufferedStream); - if (LOG_STREAMS_UPLOAD.isDebugEnabled()) { - // Log message, with exception so we can get a trace to see where the call came from - LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure Blob Storage - identifier={}", key, new Exception()); + boolean useBufferedStream = len < AzureConstantsV8.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD; + final InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file); + try { + blob.upload(in, len, null, options, null); + LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, (System.currentTimeMillis() - start), useBufferedStream); + if (LOG_STREAMS_UPLOAD.isDebugEnabled()) { + // Log message, with exception so we can get a trace to see where the call came from + LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure Blob Storage - identifier={}", key, new Exception()); + } + } finally { + in.close(); } - } return; } blob.downloadAttributes(); if (blob.getProperties().getLength() != len) { throw new DataStoreException("Length Collision. identifier=" + key + - " new length=" + len + - " old length=" + blob.getProperties().getLength()); + " new length=" + len + + " old length=" + blob.getProperties().getLength()); } LOG.trace("Blob already exists. identifier={} lastModified={}", key, getLastModified(blob)); @@ -345,24 +342,81 @@ public void write(DataIdentifier identifier, File file) throws DataStoreExceptio blob.uploadMetadata(); LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key, - getLastModified(blob), watch.elapsed(TimeUnit.MILLISECONDS)); + getLastModified(blob), (System.currentTimeMillis() - start)); } - catch (StorageException | URISyntaxException | IOException e) { + catch (StorageException e) { + LOG.info("Error writing blob. identifier={}", key, e); + throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e); + } + catch (URISyntaxException | IOException e) { LOG.debug("Error writing blob. identifier={}", key, e); throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e); } finally { - if (contextClassLoader != null) { + if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } } - + + private static boolean waitForCopy(CloudBlob blob) throws StorageException, InterruptedException { + boolean continueLoop = true; + CopyStatus status = CopyStatus.PENDING; + while (continueLoop) { + blob.downloadAttributes(); + status = blob.getCopyState().getStatus(); + continueLoop = status == CopyStatus.PENDING; + // Sleep if retry is needed + if (continueLoop) { + Thread.sleep(500); + } + } + return status == CopyStatus.SUCCESS; + } + @Override - public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException { - Objects.requireNonNull(identifier, "identifier must not be null"); + public byte[] getOrCreateReferenceKey() throws DataStoreException { + try { + if (secret != null && secret.length != 0) { + return secret; + } else { + byte[] key; + // Try reading from the metadata folder if it exists + key = readMetadataBytes(AzureConstantsV8.AZURE_BLOB_REF_KEY); + if (key == null) { + key = super.getOrCreateReferenceKey(); + addMetadataRecord(new ByteArrayInputStream(key), AzureConstantsV8.AZURE_BLOB_REF_KEY); + key = readMetadataBytes(AzureConstantsV8.AZURE_BLOB_REF_KEY); + } + secret = key; + return secret; + } + } catch (IOException e) { + throw new DataStoreException("Unable to get or create key " + e); + } + } + + private byte[] readMetadataBytes(String name) throws IOException, DataStoreException { + DataRecord rec = getMetadataRecord(name); + byte[] key = null; + if (rec != null) { + InputStream stream = null; + try { + stream = rec.getStream(); + return IOUtils.toByteArray(stream); + } finally { + IOUtils.closeQuietly(stream); + } + } + return key; + } + @Override + public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException { + if (null == identifier) { + throw new NullPointerException("identifier"); + } String key = getKeyName(identifier); - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); @@ -376,7 +430,7 @@ public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException getLastModified(blob), blob.getProperties().getLength()); LOG.debug("Data record read for blob. identifier={} duration={} record={}", - key, watch.elapsed(TimeUnit.MILLISECONDS), record); + key, (System.currentTimeMillis() - start), record); return record; } catch (StorageException e) { @@ -404,9 +458,11 @@ public Iterator getAllIdentifiers() { input -> new DataIdentifier(getIdentifierName(input.getName()))); } + + @Override public Iterator getAllRecords() { - final AbstractSharedBackend backend = this; + final org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend backend = this; return new RecordsIterator<>( input -> new AzureBlobStoreDataRecord( backend, @@ -419,40 +475,38 @@ public Iterator getAllRecords() { @Override public boolean exists(DataIdentifier identifier) throws DataStoreException { - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); String key = getKeyName(identifier); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - boolean exists = getAzureContainer().getBlockBlobReference(key).exists(); - LOG.debug("Blob exists={} identifier={} duration={}", exists, key, watch.elapsed(TimeUnit.MILLISECONDS)); + boolean exists =getAzureContainer().getBlockBlobReference(key).exists(); + LOG.debug("Blob exists={} identifier={} duration={}", exists, key, (System.currentTimeMillis() - start)); return exists; } catch (Exception e) { throw new DataStoreException(e); } finally { - if (contextClassLoader != null) { + if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } } @Override - public void close() { - if(azureBlobContainerProvider != null) { - azureBlobContainerProvider.close(); - } + public void close() throws DataStoreException { + azureBlobContainerProvider.close(); LOG.info("AzureBlobBackend closed."); } @Override public void deleteRecord(DataIdentifier identifier) throws DataStoreException { - Objects.requireNonNull(identifier, "identifier must not be null"); + if (null == identifier) throw new NullPointerException("identifier"); String key = getKeyName(identifier); - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); @@ -460,10 +514,13 @@ public void deleteRecord(DataIdentifier identifier) throws DataStoreException { boolean result = getAzureContainer().getBlockBlobReference(key).deleteIfExists(); LOG.debug("Blob {}. identifier={} duration={}", result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)", - key, watch.elapsed(TimeUnit.MILLISECONDS)); + key, (System.currentTimeMillis() - start)); } - catch (StorageException | URISyntaxException e) { - LOG.error("Error deleting blob. identifier={}", key, e); + catch (StorageException e) { + LOG.info("Error deleting blob. identifier={}", key, e); + throw new DataStoreException(e); + } + catch (URISyntaxException e) { throw new DataStoreException(e); } finally { if (contextClassLoader != null) { @@ -474,19 +531,22 @@ public void deleteRecord(DataIdentifier identifier) throws DataStoreException { @Override public void addMetadataRecord(InputStream input, String name) throws DataStoreException { - Objects.requireNonNull(input, "input must not be null"); - Validate.checkArgument(StringUtils.isNotEmpty(name), "name should not be empty"); - - Stopwatch watch = Stopwatch.createStarted(); + if (null == input) { + throw new NullPointerException("input"); + } + if (StringUtils.isEmpty(name)) { + throw new IllegalArgumentException("name"); + } + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); addMetadataRecordImpl(input, name, -1L); - LOG.debug("Metadata record added. metadataName={} duration={}", name, watch.elapsed(TimeUnit.MILLISECONDS)); + LOG.debug("Metadata record added. metadataName={} duration={}", name, (System.currentTimeMillis() - start)); } finally { - if (contextClassLoader != null) { + if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } @@ -494,23 +554,25 @@ public void addMetadataRecord(InputStream input, String name) throws DataStoreEx @Override public void addMetadataRecord(File input, String name) throws DataStoreException { - Objects.requireNonNull(input, "input must not be null"); - Validate.checkArgument(StringUtils.isNotEmpty(name), "name should not be empty"); - - Stopwatch watch = Stopwatch.createStarted(); + if (null == input) { + throw new NullPointerException("input"); + } + if (StringUtils.isEmpty(name)) { + throw new IllegalArgumentException("name"); + } + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); addMetadataRecordImpl(new FileInputStream(input), name, input.length()); - LOG.debug("Metadata record added. metadataName={} duration={}", name, watch.elapsed(TimeUnit.MILLISECONDS)); + LOG.debug("Metadata record added. metadataName={} duration={}", name, (System.currentTimeMillis() - start)); } catch (FileNotFoundException e) { - LOG.error("Error adding metadata record. metadataName={}", name, e); throw new DataStoreException(e); } finally { - if (contextClassLoader != null) { + if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } @@ -518,13 +580,16 @@ public void addMetadataRecord(File input, String name) throws DataStoreException private void addMetadataRecordImpl(final InputStream input, String name, long recordLength) throws DataStoreException { try { - CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME); + CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AzureConstantsV8.AZURE_BLOB_META_DIR_NAME); CloudBlockBlob blob = metaDir.getBlockBlobReference(name); addLastModified(blob); blob.upload(input, recordLength); } - catch (StorageException | URISyntaxException | IOException e) { - LOG.error("Error adding metadata record. metadataName={} length={}", name, recordLength, e); + catch (StorageException e) { + LOG.info("Error adding metadata record. metadataName={} length={}", name, recordLength, e); + throw new DataStoreException(e); + } + catch (URISyntaxException | IOException e) { throw new DataStoreException(e); } } @@ -532,11 +597,11 @@ private void addMetadataRecordImpl(final InputStream input, String name, long re @Override public DataRecord getMetadataRecord(String name) { ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME); + CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AzureConstantsV8.AZURE_BLOB_META_DIR_NAME); CloudBlockBlob blob = metaDir.getBlockBlobReference(name); if (!blob.exists()) { LOG.warn("Trying to read missing metadata. metadataName={}", name); @@ -551,14 +616,17 @@ public DataRecord getMetadataRecord(String name) { lastModified, length, true); - LOG.debug("Metadata record read. metadataName={} duration={} record={}", name, watch.elapsed(TimeUnit.MILLISECONDS), record); + LOG.debug("Metadata record read. metadataName={} duration={} record={}", name, (System.currentTimeMillis() - start), record); return record; - } catch (StorageException | DataStoreException | URISyntaxException e) { + } catch (StorageException e) { LOG.info("Error reading metadata record. metadataName={}", name, e); throw new RuntimeException(e); + } catch (Exception e) { + LOG.debug("Error reading metadata record. metadataName={}", name, e); + throw new RuntimeException(e); } finally { - if (contextClassLoader != null) { + if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } @@ -566,15 +634,16 @@ public DataRecord getMetadataRecord(String name) { @Override public List getAllMetadataRecords(String prefix) { - Objects.requireNonNull(prefix, "prefix must not be null"); - - Stopwatch watch = Stopwatch.createStarted(); + if (null == prefix) { + throw new NullPointerException("prefix"); + } + long start = System.currentTimeMillis(); final List records = new ArrayList<>(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME); + CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AzureConstantsV8.AZURE_BLOB_META_DIR_NAME); for (ListBlobItem item : metaDir.listBlobs(prefix)) { if (item instanceof CloudBlob) { CloudBlob blob = (CloudBlob) item; @@ -588,13 +657,16 @@ public List getAllMetadataRecords(String prefix) { true)); } } - LOG.debug("Metadata records read. recordsRead={} metadataFolder={} duration={}", records.size(), prefix, watch.elapsed(TimeUnit.MILLISECONDS)); + LOG.debug("Metadata records read. recordsRead={} metadataFolder={} duration={}", records.size(), prefix, (System.currentTimeMillis() - start)); + } + catch (StorageException e) { + LOG.info("Error reading all metadata records. metadataFolder={}", prefix, e); } - catch (StorageException | DataStoreException | URISyntaxException e) { - LOG.error("Error reading all metadata records. metadataFolder={}", prefix, e); + catch (DataStoreException | URISyntaxException e) { + LOG.debug("Error reading all metadata records. metadataFolder={}", prefix, e); } finally { - if (contextClassLoader != null) { + if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } @@ -603,7 +675,7 @@ public List getAllMetadataRecords(String prefix) { @Override public boolean deleteMetadataRecord(String name) { - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); @@ -612,12 +684,15 @@ public boolean deleteMetadataRecord(String name) { boolean result = blob.deleteIfExists(); LOG.debug("Metadata record {}. metadataName={} duration={}", result ? "deleted" : "delete requested, but it does not exist (perhaps already deleted)", - name, watch.elapsed(TimeUnit.MILLISECONDS)); + name, (System.currentTimeMillis() - start)); return result; } - catch (StorageException | DataStoreException | URISyntaxException e) { - LOG.error("Error deleting metadata record. metadataName={}", name, e); + catch (StorageException e) { + LOG.info("Error deleting metadata record. metadataName={}", name, e); + } + catch (DataStoreException | URISyntaxException e) { + LOG.debug("Error deleting metadata record. metadataName={}", name, e); } finally { if (contextClassLoader != null) { @@ -629,14 +704,15 @@ public boolean deleteMetadataRecord(String name) { @Override public void deleteAllMetadataRecords(String prefix) { - Objects.requireNonNull(prefix, "prefix must not be null"); - - Stopwatch watch = Stopwatch.createStarted(); + if (null == prefix) { + throw new NullPointerException("prefix"); + } + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME); + CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AzureConstantsV8.AZURE_BLOB_META_DIR_NAME); int total = 0; for (ListBlobItem item : metaDir.listBlobs(prefix)) { if (item instanceof CloudBlob) { @@ -646,14 +722,17 @@ public void deleteAllMetadataRecords(String prefix) { } } LOG.debug("Metadata records deleted. recordsDeleted={} metadataFolder={} duration={}", - total, prefix, watch.elapsed(TimeUnit.MILLISECONDS)); + total, prefix, (System.currentTimeMillis() - start)); } - catch (StorageException | DataStoreException | URISyntaxException e) { + catch (StorageException e) { + LOG.info("Error deleting all metadata records. metadataFolder={}", prefix, e); + } + catch (DataStoreException | URISyntaxException e) { LOG.debug("Error deleting all metadata records. metadataFolder={}", prefix, e); } finally { - if (contextClassLoader != null) { + if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } @@ -661,13 +740,13 @@ public void deleteAllMetadataRecords(String prefix) { @Override public boolean metadataRecordExists(String name) { - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(addMetaKeyPrefix(name)); boolean exists = blob.exists(); - LOG.debug("Metadata record {} exists {}. duration={}", name, exists, watch.elapsed(TimeUnit.MILLISECONDS)); + LOG.debug("Metadata record {} exists {}. duration={}", name, exists, (System.currentTimeMillis() - start)); return exists; } catch (DataStoreException | StorageException | URISyntaxException e) { @@ -681,6 +760,7 @@ public boolean metadataRecordExists(String name) { return false; } + /** * Get key from data identifier. Object is stored with key in ADS. */ @@ -695,39 +775,39 @@ private static String getKeyName(DataIdentifier identifier) { private static String getIdentifierName(String key) { if (!key.contains(UtilsV8.DASH)) { return null; - } else if (key.contains(AZURE_BLOB_META_KEY_PREFIX)) { + } else if (key.contains(AzureConstantsV8.AZURE_BLOB_META_KEY_PREFIX)) { return key; } return key.substring(0, 4) + key.substring(5); } private static String addMetaKeyPrefix(final String key) { - return AZURE_BLOB_META_KEY_PREFIX + key; + return AzureConstantsV8.AZURE_BLOB_META_KEY_PREFIX + key; } private static String stripMetaKeyPrefix(String name) { - if (name.startsWith(AZURE_BLOB_META_KEY_PREFIX)) { - return name.substring(AZURE_BLOB_META_KEY_PREFIX.length()); + if (name.startsWith(AzureConstantsV8.AZURE_BLOB_META_KEY_PREFIX)) { + return name.substring(AzureConstantsV8.AZURE_BLOB_META_KEY_PREFIX.length()); } return name; } private static void addLastModified(CloudBlockBlob blob) { - blob.getMetadata().put(AZURE_BLOB_LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis())); + blob.getMetadata().put(AzureConstantsV8.AZURE_BLOB_LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis())); } private static long getLastModified(CloudBlob blob) { - if (blob.getMetadata().containsKey(AZURE_BLOB_LAST_MODIFIED_KEY)) { - return Long.parseLong(blob.getMetadata().get(AZURE_BLOB_LAST_MODIFIED_KEY)); + if (blob.getMetadata().containsKey(AzureConstantsV8.AZURE_BLOB_LAST_MODIFIED_KEY)) { + return Long.parseLong(blob.getMetadata().get(AzureConstantsV8.AZURE_BLOB_LAST_MODIFIED_KEY)); } return blob.getProperties().getLastModified().getTime(); } - protected void setHttpDownloadURIExpirySeconds(int seconds) { + public void setHttpDownloadURIExpirySeconds(int seconds) { httpDownloadURIExpirySeconds = seconds; } - protected void setHttpDownloadURICacheSize(int maxSize) { + public void setHttpDownloadURICacheSize(int maxSize) { // max size 0 or smaller is used to turn off the cache if (maxSize > 0) { LOG.info("presigned GET URI cache enabled, maxSize = {} items, expiry = {} seconds", maxSize, httpDownloadURIExpirySeconds / 2); @@ -741,27 +821,29 @@ protected void setHttpDownloadURICacheSize(int maxSize) { } } - @Override - protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier, + public URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions) { URI uri = null; - Objects.requireNonNull(identifier, "identifier must not be null"); - Objects.requireNonNull(downloadOptions, "downloadOptions must not be null"); + // When running unit test from Maven, it doesn't always honor the @NotNull decorators + if (null == identifier) throw new NullPointerException("identifier"); + if (null == downloadOptions) throw new NullPointerException("downloadOptions"); if (httpDownloadURIExpirySeconds > 0) { String domain = getDirectDownloadBlobStorageDomain(downloadOptions.isDomainOverrideIgnored()); - Objects.requireNonNull(domain, "Could not determine domain for direct download"); + if (null == domain) { + throw new NullPointerException("Could not determine domain for direct download"); + } - String cacheKey = identifier + String cacheKey = identifier.toString() + domain + Objects.toString(downloadOptions.getContentTypeHeader(), "") + Objects.toString(downloadOptions.getContentDispositionHeader(), ""); - if (httpDownloadURICache != null) { + if (null != httpDownloadURICache) { uri = httpDownloadURICache.getIfPresent(cacheKey); } - if (uri == null) { + if (null == uri) { if (presignedDownloadURIVerifyExists) { // Check if this identifier exists. If not, we want to return null // even if the identifier is in the download URI cache. @@ -781,13 +863,13 @@ protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier, headers.setCacheControl(String.format("private, max-age=%d, immutable", httpDownloadURIExpirySeconds)); String contentType = downloadOptions.getContentTypeHeader(); - if (!Objects.toString(contentType, "").isEmpty()) { + if (! StringUtils.isEmpty(contentType)) { headers.setContentType(contentType); } String contentDisposition = downloadOptions.getContentDispositionHeader(); - if (!Objects.toString(contentDisposition, "").isEmpty()) { + if (! StringUtils.isEmpty(contentDisposition)) { headers.setContentDisposition(contentDisposition); } @@ -804,7 +886,7 @@ protected URI createHttpDownloadURI(@NotNull DataIdentifier identifier, return uri; } - protected void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; } + public void setHttpUploadURIExpirySeconds(int seconds) { httpUploadURIExpirySeconds = seconds; } private DataIdentifier generateSafeRandomIdentifier() { return new DataIdentifier( @@ -815,15 +897,35 @@ private DataIdentifier generateSafeRandomIdentifier() { ); } - protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) { + public DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) { List uploadPartURIs = new ArrayList<>(); - long minPartSize = AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE; - long maxPartSize = AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; + long minPartSize = AzureConstantsV8.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE; + long maxPartSize = AzureConstantsV8.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; - Validate.checkArgument(maxUploadSizeInBytes > 0L, "maxUploadSizeInBytes must be > 0"); - Validate.checkArgument(maxNumberOfURIs > 0 || maxNumberOfURIs == -1, "maxNumberOfURIs must either be > 0 or -1"); - Validate.checkArgument(!(maxUploadSizeInBytes > AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE && maxNumberOfURIs == 1), "Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE); - Validate.checkArgument(maxUploadSizeInBytes <= AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE, "Cannot do upload with file size %d - exceeds max upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE); + if (0L >= maxUploadSizeInBytes) { + throw new IllegalArgumentException("maxUploadSizeInBytes must be > 0"); + } + else if (0 == maxNumberOfURIs) { + throw new IllegalArgumentException("maxNumberOfURIs must either be > 0 or -1"); + } + else if (-1 > maxNumberOfURIs) { + throw new IllegalArgumentException("maxNumberOfURIs must either be > 0 or -1"); + } + else if (maxUploadSizeInBytes > AzureConstantsV8.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE && + maxNumberOfURIs == 1) { + throw new IllegalArgumentException( + String.format("Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d", + maxUploadSizeInBytes, + AzureConstantsV8.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE) + ); + } + else if (maxUploadSizeInBytes > AzureConstantsV8.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE) { + throw new IllegalArgumentException( + String.format("Cannot do upload with file size %d - exceeds max upload size of %d", + maxUploadSizeInBytes, + AzureConstantsV8.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE) + ); + } DataIdentifier newIdentifier = generateSafeRandomIdentifier(); String blobId = getKeyName(newIdentifier); @@ -856,7 +958,7 @@ protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int max maxNumberOfURIs, Math.min( (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) minPartSize)), - AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS + AzureConstantsV8.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS ) ); } else { @@ -866,13 +968,15 @@ protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int max } } else { - long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); - numParts = Math.min(maximalNumParts, AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS); + long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) AzureConstantsV8.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); + numParts = Math.min(maximalNumParts, AzureConstantsV8.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS); } String key = getKeyName(newIdentifier); String domain = getDirectUploadBlobStorageDomain(options.isDomainOverrideIgnored()); - Objects.requireNonNull(domain, "Could not determine domain for direct upload"); + if (null == domain) { + throw new NullPointerException("Could not determine domain for direct upload"); + } EnumSet perms = EnumSet.of(SharedAccessBlobPermissions.WRITE); Map presignedURIRequestParams = new HashMap<>(); @@ -924,10 +1028,12 @@ public Collection getUploadURIs() { return null; } - protected DataRecord completeHttpUpload(@NotNull String uploadTokenStr) + public DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException { - Validate.checkArgument(StringUtils.isNotEmpty(uploadTokenStr), "uploadToken required"); + if (StringUtils.isEmpty(uploadTokenStr)) { + throw new IllegalArgumentException("uploadToken required"); + } DataRecordUploadToken uploadToken = DataRecordUploadToken.fromEncodedToken(uploadTokenStr, getOrCreateReferenceKey()); String key = uploadToken.getBlobId(); @@ -984,8 +1090,8 @@ record = new AzureBlobStoreDataRecord( } private String getDefaultBlobStorageDomain() { - String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); - if (Objects.toString(accountName, "").isEmpty()) { + String accountName = properties.getProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, ""); + if (StringUtils.isEmpty(accountName)) { LOG.warn("Can't generate presigned URI - Azure account name not found in properties"); return null; } @@ -996,7 +1102,7 @@ private String getDirectDownloadBlobStorageDomain(boolean ignoreDomainOverride) String domain = ignoreDomainOverride ? getDefaultBlobStorageDomain() : downloadDomainOverride; - if (Objects.toString(domain, "").isEmpty()) { + if (StringUtils.isEmpty(domain)) { domain = getDefaultBlobStorageDomain(); } return domain; @@ -1006,7 +1112,7 @@ private String getDirectUploadBlobStorageDomain(boolean ignoreDomainOverride) { String domain = ignoreDomainOverride ? getDefaultBlobStorageDomain() : uploadDomainOverride; - if (Objects.toString(domain, "").isEmpty()) { + if (StringUtils.isEmpty(domain)) { domain = getDefaultBlobStorageDomain(); } return domain; @@ -1034,7 +1140,7 @@ private URI createPresignedURI(String key, Map additionalQueryParams, SharedAccessBlobHeaders optionalHeaders, String domain) { - if (Objects.toString(domain, "").isEmpty()) { + if (StringUtils.isEmpty(domain)) { LOG.warn("Can't generate presigned URI - no Azure domain provided (is Azure account name configured?)"); return null; } @@ -1065,7 +1171,7 @@ private URI createPresignedURI(String key, presignedURI = new URI(uriString); } catch (DataStoreException e) { - LOG.error("Error calling Azure Blob Storage", e); + LOG.error("No connection to Azure Blob Storage", e); } catch (URISyntaxException | InvalidKeyException e) { LOG.error("Can't generate a presigned URI for key {}", key, e); @@ -1110,8 +1216,8 @@ public long getLength() { public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) throws StorageException { cloudBlob.downloadAttributes(); return new AzureBlobInfo(cloudBlob.getName(), - AzureBlobStoreBackendV8.getLastModified(cloudBlob), - cloudBlob.getProperties().getLength()); + AzureBlobStoreBackendV8.getLastModified(cloudBlob), + cloudBlob.getProperties().getLength()); } } @@ -1138,7 +1244,7 @@ protected T computeNext() { } private boolean loadItems() { - Stopwatch watch = Stopwatch.createStarted(); + long start = System.currentTimeMillis(); ClassLoader contextClassLoader = currentThread().getContextClassLoader(); try { currentThread().setContextClassLoader(getClass().getClassLoader()); @@ -1157,11 +1263,14 @@ private boolean loadItems() { } } LOG.debug("Container records batch read. batchSize={} containerName={} duration={}", - results.getLength(), getContainerName(), watch.elapsed(TimeUnit.MILLISECONDS)); + results.getLength(), getContainerName(), (System.currentTimeMillis() - start)); return results.getLength() > 0; } - catch (StorageException | DataStoreException e) { - LOG.error("Error listing blobs. containerName={}", getContainerName(), e); + catch (StorageException e) { + LOG.info("Error listing blobs. containerName={}", getContainerName(), e); + } + catch (DataStoreException e) { + LOG.debug("Cannot list blobs. containerName={}", getContainerName(), e); } finally { if (contextClassLoader != null) { currentThread().setContextClassLoader(contextClassLoader); @@ -1177,12 +1286,12 @@ static class AzureBlobStoreDataRecord extends AbstractDataRecord { final long length; final boolean isMeta; - public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider, + public AzureBlobStoreDataRecord(org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider, DataIdentifier key, long lastModified, long length) { this(backend, azureBlobContainerProvider, key, lastModified, length, false); } - public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider, + public AzureBlobStoreDataRecord(org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend backend, AzureBlobContainerProviderV8 azureBlobContainerProvider, DataIdentifier key, long lastModified, long length, boolean isMeta) { super(backend, key); this.azureBlobContainerProvider = azureBlobContainerProvider; @@ -1192,7 +1301,7 @@ public AzureBlobStoreDataRecord(AbstractSharedBackend backend, AzureBlobContaine } @Override - public long getLength() { + public long getLength() throws DataStoreException { return length; } @@ -1225,11 +1334,11 @@ public long getLastModified() { @Override public String toString() { return "AzureBlobStoreDataRecord{" + - "identifier=" + getIdentifier() + - ", length=" + length + - ", lastModified=" + lastModified + - ", containerName='" + Optional.ofNullable(azureBlobContainerProvider).map(AzureBlobContainerProviderV8::getContainerName).orElse(null) + '\'' + - '}'; + "identifier=" + getIdentifier() + + ", length=" + length + + ", lastModified=" + lastModified + + ", containerName='" + Optional.ofNullable(azureBlobContainerProvider).map(AzureBlobContainerProviderV8::getContainerName).orElse(null) + '\'' + + '}'; } } @@ -1238,42 +1347,4 @@ private String getContainerName() { .map(AzureBlobContainerProviderV8::getContainerName) .orElse(null); } - - @Override - public byte[] getOrCreateReferenceKey() throws DataStoreException { - try { - if (secret != null && secret.length != 0) { - return secret; - } else { - byte[] key; - // Try reading from the metadata folder if it exists - key = readMetadataBytes(AZURE_BLOB_REF_KEY); - if (key == null) { - key = super.getOrCreateReferenceKey(); - addMetadataRecord(new ByteArrayInputStream(key), AZURE_BLOB_REF_KEY); - key = readMetadataBytes(AZURE_BLOB_REF_KEY); - } - secret = key; - return secret; - } - } catch (IOException e) { - throw new DataStoreException("Unable to get or create key " + e); - } - } - - protected byte[] readMetadataBytes(String name) throws IOException, DataStoreException { - DataRecord rec = getMetadataRecord(name); - byte[] key = null; - if (rec != null) { - InputStream stream = null; - try { - stream = rec.getStream(); - return IOUtils.toByteArray(stream); - } finally { - IOUtils.closeQuietly(stream); - } - } - return key; - } - } diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureConstantsV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureConstantsV8.java new file mode 100644 index 00000000000..228f5051854 --- /dev/null +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureConstantsV8.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8; + +/** + * Constants for the Azure Blob Storage v8 SDK backend. + *

+ * These values are self-contained and match the constants shipped in Oak 1.90.0. + * They must not reference or delegate to the shared {@code AzureConstants} class, + * so that the v8 package remains an exact behavioural match to the pre-refactoring code. + */ +public final class AzureConstantsV8 { + + // --- Configuration property names (match 1.90.0 AzureConstants) --- + + public static final String AZURE_STORAGE_ACCOUNT_NAME = "accessKey"; + public static final String AZURE_STORAGE_ACCOUNT_KEY = "secretKey"; + public static final String AZURE_CONNECTION_STRING = "azureConnectionString"; + public static final String AZURE_SAS = "azureSas"; + public static final String AZURE_TENANT_ID = "tenantId"; + public static final String AZURE_CLIENT_ID = "clientId"; + public static final String AZURE_CLIENT_SECRET = "clientSecret"; + public static final String AZURE_BLOB_ENDPOINT = "azureBlobEndpoint"; + public static final String AZURE_BLOB_CONTAINER_NAME = "container"; + public static final String AZURE_CREATE_CONTAINER = "azureCreateContainer"; + public static final String AZURE_BLOB_REQUEST_TIMEOUT = "socketTimeout"; + public static final String AZURE_BLOB_MAX_REQUEST_RETRY = "maxErrorRetry"; + public static final String AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION = "maxConnections"; + public static final String AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME = "enableSecondaryLocation"; + public static final boolean AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT = false; + public static final String PROXY_HOST = "proxyHost"; + public static final String PROXY_PORT = "proxyPort"; + public static final String PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS = "presignedHttpUploadURIExpirySeconds"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS = "presignedHttpDownloadURIExpirySeconds"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE = "presignedHttpDownloadURICacheMaxSize"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS = "presignedHttpDownloadURIVerifyExists"; + public static final String PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpDownloadURIDomainOverride"; + public static final String PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpUploadURIDomainOverride"; + public static final String AZURE_REF_ON_INIT = "refOnInit"; + + // --- v8-specific runtime constants (match 1.90.0 AzureBlobStoreBackend) --- + // These values reflect Azure SDK v8 limits. See AzureConstantsV12 for the v12 equivalents. + + public static final String AZURE_BLOB_META_DIR_NAME = "META"; + public static final String AZURE_BLOB_META_KEY_PREFIX = AZURE_BLOB_META_DIR_NAME + "/"; + public static final String AZURE_BLOB_REF_KEY = "reference.key"; + public static final String AZURE_BLOB_LAST_MODIFIED_KEY = "lastModified"; + public static final long AZURE_BLOB_BUFFERED_STREAM_THRESHOLD = 1024L * 1024L; // 1 MiB + public static final long AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE = 1024L * 1024L * 10L; // 10 MiB + public static final long AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE = 1024L * 1024L * 100L; // 100 MiB (SDK v8 maximum block size) + public static final long AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE = 1024L * 1024L * 256L; // 256 MiB + public static final long AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE = + (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // ~4.75 TiB (50,000 blocks x 100 MiB) + public static final int AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS = 50000; + public static final int AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES = 10; + public static final int AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT = 2; + public static final int AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT = 50; + + private AzureConstantsV8() { + } +} diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java index c0b5e92cc04..2c5991d8d6d 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8.java @@ -19,6 +19,17 @@ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.SocketAddress; +import java.net.URISyntaxException; +import java.security.InvalidKeyException; +import java.util.Properties; + import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.OperationContext; import com.microsoft.azure.storage.RetryExponentialRetry; @@ -30,19 +41,10 @@ import com.microsoft.azure.storage.blob.CloudBlobContainer; import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.net.InetSocketAddress; -import java.net.Proxy; -import java.net.SocketAddress; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; -import java.util.Objects; -import java.util.Properties; - public final class UtilsV8 { public static final String DEFAULT_CONFIG_FILE = "azure.properties"; @@ -69,7 +71,7 @@ public static CloudBlobClient getBlobClient(@NotNull final String connectionStri @Nullable final BlobRequestOptions requestOptions) throws URISyntaxException, InvalidKeyException { CloudStorageAccount account = CloudStorageAccount.parse(connectionString); CloudBlobClient client = account.createCloudBlobClient(); - if (requestOptions != null) { + if (null != requestOptions) { client.setDefaultRequestOptions(requestOptions); } return client; @@ -80,13 +82,12 @@ public static CloudBlobContainer getBlobContainer(@NotNull final String connecti return getBlobContainer(connectionString, containerName, null); } - public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString, @NotNull final String containerName, @Nullable final BlobRequestOptions requestOptions) throws DataStoreException { try { CloudBlobClient client = ( - (requestOptions == null) + (null == requestOptions) ? UtilsV8.getBlobClient(connectionString) : UtilsV8.getBlobClient(connectionString, requestOptions) ); @@ -97,11 +98,11 @@ public static CloudBlobContainer getBlobContainer(@NotNull final String connecti } public static void setProxyIfNeeded(final Properties properties) { - String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST); - String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT); + String proxyHost = properties.getProperty(AzureConstantsV8.PROXY_HOST); + String proxyPort = properties.getProperty(AzureConstantsV8.PROXY_PORT); - if (!(Objects.toString(proxyHost, "").isEmpty() || - Objects.toString(proxyPort, "").isEmpty())) { + if (!StringUtils.isEmpty(proxyHost) && + !StringUtils.isEmpty(proxyPort)) { int port = Integer.parseInt(proxyPort); SocketAddress proxyAddr = new InetSocketAddress(proxyHost, port); Proxy proxy = new Proxy(Proxy.Type.HTTP, proxyAddr); @@ -109,13 +110,25 @@ public static void setProxyIfNeeded(final Properties properties) { } } + public static RetryPolicy getRetryPolicy(final String maxRequestRetry) { + int retries = PropertiesUtil.toInteger(maxRequestRetry, -1); + if (retries < 0) { + return null; + } + if (retries == 0) { + return new RetryNoRetry(); + } + return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries); + } + + public static String getConnectionStringFromProperties(Properties properties) { - String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, ""); - String blobEndpoint = properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, ""); - String connectionString = properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, ""); - String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); - String accountKey = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, ""); + String sasUri = properties.getProperty(AzureConstantsV8.AZURE_SAS, ""); + String blobEndpoint = properties.getProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, ""); + String connectionString = properties.getProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, ""); + String accountName = properties.getProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, ""); + String accountKey = properties.getProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, ""); if (!connectionString.isEmpty()) { return connectionString; @@ -127,7 +140,7 @@ public static String getConnectionStringFromProperties(Properties properties) { return getConnectionString( accountName, - accountKey, + accountKey, blobEndpoint); } @@ -142,26 +155,64 @@ public static String getConnectionStringForSas(String sasUri, String blobEndpoin public static String getConnectionString(final String accountName, final String accountKey) { return getConnectionString(accountName, accountKey, null); } - + public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) { StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https"); connString.append(";AccountName=").append(accountName); connString.append(";AccountKey=").append(accountKey); - - if (!Objects.toString(blobEndpoint, "").isEmpty()) { + + if (!StringUtils.isEmpty(blobEndpoint)) { connString.append(";BlobEndpoint=").append(blobEndpoint); } return connString.toString(); } - public static RetryPolicy getRetryPolicy(final String maxRequestRetry) { - int retries = PropertiesUtil.toInteger(maxRequestRetry, -1); - if (retries < 0) { - return null; + /** + * Check whether the given properties contain sufficient Azure configuration + * for V8 SDK connectivity (account key, SAS, or AAD credentials). + */ + public static boolean isConfigured(Properties props) { + // Account key auth + if (props.containsKey(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY) + && props.containsKey(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME) + && props.containsKey(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME)) { + return true; } - else if (retries == 0) { - return new RetryNoRetry(); + // SAS auth + if (props.containsKey(AzureConstantsV8.AZURE_SAS) + && props.containsKey(AzureConstantsV8.AZURE_BLOB_ENDPOINT) + && props.containsKey(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME)) { + return true; } - return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries); + // AAD client credentials + return props.containsKey(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME) + && props.containsKey(AzureConstantsV8.AZURE_TENANT_ID) + && props.containsKey(AzureConstantsV8.AZURE_CLIENT_ID) + && props.containsKey(AzureConstantsV8.AZURE_CLIENT_SECRET) + && props.containsKey(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME); + } + + /** + * Read a configuration properties file. + * + * @param fileName the properties file name + * @return the properties + * @throws java.io.IOException if the file doesn't exist + */ + public static Properties readConfig(String fileName) throws IOException { + if (!new File(fileName).exists()) { + throw new IOException("Config file not found. fileName=" + fileName); + } + Properties prop = new Properties(); + InputStream in = null; + try { + in = new FileInputStream(fileName); + prop.load(in); + } finally { + if (in != null) { + in.close(); + } + } + return prop; } } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendCompatibilityTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendCompatibilityTest.java index 053ae0df50e..d362f076b52 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendCompatibilityTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendCompatibilityTest.java @@ -30,7 +30,7 @@ /** * Compatibility tests for direct-download and upload cache configuration in - * {@link AzureBlobStoreBackend}. The assertions are intentionally behavior- + * {@link org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12}. The assertions are intentionally behavior- * based and do not depend on a specific cache library. */ public class AzureBlobStoreBackendCompatibilityTest { @@ -38,7 +38,7 @@ public class AzureBlobStoreBackendCompatibilityTest { @Test public void setHttpDownloadURIExpirySecondsUpdatesField() throws Exception { // Setter coverage for the direct-download expiry value used by presigned URIs. - AzureBlobStoreBackend backend = new AzureBlobStoreBackend(); + org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12 backend = new org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12(); backend.setHttpDownloadURIExpirySeconds(3600); @@ -48,7 +48,7 @@ public void setHttpDownloadURIExpirySecondsUpdatesField() throws Exception { @Test public void setHttpUploadURIExpirySecondsUpdatesField() throws Exception { // Setter coverage for the direct-upload expiry used during upload initiation. - AzureBlobStoreBackend backend = new AzureBlobStoreBackend(); + org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12 backend = new org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12(); backend.setHttpUploadURIExpirySeconds(1800); @@ -58,7 +58,7 @@ public void setHttpUploadURIExpirySecondsUpdatesField() throws Exception { @Test public void setHttpDownloadURICacheSizeCreatesAndDisablesCache() throws Exception { // Verify the cache-size toggle actually creates and then removes the backing cache. - AzureBlobStoreBackend backend = new AzureBlobStoreBackend(); + org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12 backend = new org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12(); backend.setHttpDownloadURIExpirySeconds(3600); backend.setHttpDownloadURICacheSize(100); @@ -71,7 +71,7 @@ public void setHttpDownloadURICacheSizeCreatesAndDisablesCache() throws Exceptio @Test public void createHttpDownloadURIReturnsNullWhenDisabled() { // With no download expiry configured, direct download access should stay disabled. - AzureBlobStoreBackend backend = new AzureBlobStoreBackend(); + org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12 backend = new org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12(); URI downloadURI = backend.createHttpDownloadURI( new DataIdentifier("test"), @@ -83,19 +83,19 @@ public void createHttpDownloadURIReturnsNullWhenDisabled() { @Test public void initiateHttpUploadReturnsNullWhenDisabled() { // Upload initiation follows the same disabled-by-default contract until configured. - AzureBlobStoreBackend backend = new AzureBlobStoreBackend(); + org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12 backend = new org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12(); assertNull(backend.initiateHttpUpload(1024, 1, DataRecordUploadOptions.DEFAULT)); } - private static int getIntField(AzureBlobStoreBackend backend, String fieldName) throws Exception { - Field field = AzureBlobStoreBackend.class.getDeclaredField(fieldName); + private static int getIntField(org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12 backend, String fieldName) throws Exception { + Field field = org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12.class.getDeclaredField(fieldName); field.setAccessible(true); return (int) field.get(backend); } - private static Object getField(AzureBlobStoreBackend backend, String fieldName) throws Exception { - Field field = AzureBlobStoreBackend.class.getDeclaredField(fieldName); + private static Object getField(org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12 backend, String fieldName) throws Exception { + Field field = org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12.class.getDeclaredField(fieldName); field.setAccessible(true); return field.get(backend); } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstantsTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstantsTest.java index f259b91ca22..ce1ae1e86f5 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstantsTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstantsTest.java @@ -1,129 +1,33 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureConstantsV8; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; -/** - * Test class for AzureConstants to ensure constant values are correct - * and that changes don't break existing functionality. - */ public class AzureConstantsTest { @Test - public void testMetadataDirectoryConstant() { - // Test that the metadata directory name constant has the expected value - assertEquals("META directory name should be 'META'", "META", AzureConstants.AZURE_BlOB_META_DIR_NAME); - - // Ensure the constant is not null or empty - assertNotNull("META directory name should not be null", AzureConstants.AZURE_BlOB_META_DIR_NAME); - assertFalse("META directory name should not be empty", AzureConstants.AZURE_BlOB_META_DIR_NAME.isEmpty()); - } - - @Test - public void testMetadataKeyPrefixConstant() { - // Test that the metadata key prefix is correctly constructed - String expectedPrefix = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/"; - assertEquals("META key prefix should be constructed correctly", expectedPrefix, AzureConstants.AZURE_BLOB_META_KEY_PREFIX); - - // Verify it ends with a slash for directory structure - assertTrue("META key prefix should end with '/'", AzureConstants.AZURE_BLOB_META_KEY_PREFIX.endsWith("/")); - - // Verify it starts with the directory name - assertTrue("META key prefix should start with directory name", - AzureConstants.AZURE_BLOB_META_KEY_PREFIX.startsWith(AzureConstants.AZURE_BlOB_META_DIR_NAME)); - } - - @Test - public void testMetadataKeyPrefixValue() { - // Test the exact expected value - assertEquals("META key prefix should be 'META/'", "META/", AzureConstants.AZURE_BLOB_META_KEY_PREFIX); - - // Ensure the constant is not null or empty - assertNotNull("META key prefix should not be null", AzureConstants.AZURE_BLOB_META_KEY_PREFIX); - assertFalse("META key prefix should not be empty", AzureConstants.AZURE_BLOB_META_KEY_PREFIX.isEmpty()); - } - - @Test - public void testBlobReferenceKeyConstant() { - // Test that the blob reference key constant has the expected value - assertEquals("Blob reference key should be 'reference.key'", "reference.key", AzureConstants.AZURE_BLOB_REF_KEY); - - // Ensure the constant is not null or empty - assertNotNull("Blob reference key should not be null", AzureConstants.AZURE_BLOB_REF_KEY); - assertFalse("Blob reference key should not be empty", AzureConstants.AZURE_BLOB_REF_KEY.isEmpty()); - } - - @Test - public void testConstantConsistency() { - // Test that the constants are consistent with each other - String expectedKeyPrefix = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/"; - assertEquals("Key prefix should be consistent with directory name", - expectedKeyPrefix, AzureConstants.AZURE_BLOB_META_KEY_PREFIX); - } - - @Test - public void testConstantImmutability() { - // Test that constants are final and cannot be changed (compile-time check) - // This test verifies that the constants exist and have expected types - assertTrue("META directory name should be a String", AzureConstants.AZURE_BlOB_META_DIR_NAME instanceof String); - assertTrue("META key prefix should be a String", AzureConstants.AZURE_BLOB_META_KEY_PREFIX instanceof String); - assertTrue("Blob reference key should be a String", AzureConstants.AZURE_BLOB_REF_KEY instanceof String); - } - - @Test - public void testMetadataPathConstruction() { - // Test that metadata paths can be constructed correctly using the constants - String testFileName = "test-metadata.json"; - String expectedPath = AzureConstants.AZURE_BLOB_META_KEY_PREFIX + testFileName; - String actualPath = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/" + testFileName; - - assertEquals("Metadata path construction should be consistent", expectedPath, actualPath); - assertEquals("Metadata path should start with META/", "META/" + testFileName, expectedPath); - } - - @Test - public void testConstantNamingConvention() { - // Test that the constant names follow expected naming conventions - - // These tests verify that the constants exist by accessing them - // If the constant names were changed incorrectly, these would fail at compile time - assertNotNull("Directory constant should exist", AzureConstants.AZURE_BlOB_META_DIR_NAME); - assertNotNull("Prefix constant should exist", AzureConstants.AZURE_BLOB_META_KEY_PREFIX); - assertNotNull("Reference key constant should exist", AzureConstants.AZURE_BLOB_REF_KEY); - } - - @Test - public void testBackwardCompatibility() { - // Test that the constant values maintain backward compatibility - // These values should not change as they affect stored data structure - assertEquals("META directory name must remain 'META' for backward compatibility", - "META", AzureConstants.AZURE_BlOB_META_DIR_NAME); - assertEquals("Reference key must remain 'reference.key' for backward compatibility", - "reference.key", AzureConstants.AZURE_BLOB_REF_KEY); - } - - @Test - public void testAllStringConstants() { - // Test all string constants have expected values + public void testSharedStringConstants() { + assertEquals("blob.azure.v12.enabled", AzureConstants.AZURE_V12_ENABLED_PROPERTY); assertEquals("accessKey", AzureConstants.AZURE_STORAGE_ACCOUNT_NAME); assertEquals("secretKey", AzureConstants.AZURE_STORAGE_ACCOUNT_KEY); assertEquals("azureConnectionString", AzureConstants.AZURE_CONNECTION_STRING); @@ -133,111 +37,56 @@ public void testAllStringConstants() { assertEquals("clientSecret", AzureConstants.AZURE_CLIENT_SECRET); assertEquals("azureBlobEndpoint", AzureConstants.AZURE_BLOB_ENDPOINT); assertEquals("container", AzureConstants.AZURE_BLOB_CONTAINER_NAME); - assertEquals("azureCreateContainer", AzureConstants.AZURE_CREATE_CONTAINER); - assertEquals("socketTimeout", AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT); - assertEquals("maxErrorRetry", AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY); - assertEquals("maxConnections", AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION); - assertEquals("enableSecondaryLocation", AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME); - assertEquals("proxyHost", AzureConstants.PROXY_HOST); - assertEquals("proxyPort", AzureConstants.PROXY_PORT); - assertEquals("lastModified", AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY); - assertEquals("refOnInit", AzureConstants.AZURE_REF_ON_INIT); } - @Test - public void testPresignedURIConstants() { - // Test presigned URI related constants - assertEquals("presignedHttpUploadURIExpirySeconds", AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS); - assertEquals("presignedHttpDownloadURIExpirySeconds", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS); - assertEquals("presignedHttpDownloadURICacheMaxSize", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE); - assertEquals("presignedHttpDownloadURIVerifyExists", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS); - assertEquals("presignedHttpDownloadURIDomainOverride", AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE); - assertEquals("presignedHttpUploadURIDomainOverride", AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE); - } + // ===================================================================== + // Cross-contamination guards (CSO Prevention) + // + // V8 and V12 runtime constants must ALL differ. If any match, + // one version's values have leaked into the other. + // These tests live in the shared parent package because they + // deliberately reference both v8 and v12 constants. + // ===================================================================== @Test - public void testNumericConstants() { - // Test all numeric constants have expected values - assertFalse("Secondary location should be disabled by default", AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT); - assertEquals("Buffered stream threshold should be 8MB", 8L * 1024L * 1024L, AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD); - assertEquals("Min multipart upload part size should be 256KB", 256L * 1024L, AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE); - assertEquals("Max multipart upload part size should be 4000MB", 4000L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE); - assertEquals("Max single PUT upload size should be 256MB", 256L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE); - assertEquals("Max binary upload size should be ~190.7TB", 190L * 1024L * 1024L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE); - assertEquals("Max allowable upload URIs should be 50000", 50000, AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS); - assertEquals("Max unique record tries should be 10", 10, AzureConstants.AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES); - assertEquals("Default concurrent request count should be 5", 5, AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); - assertEquals("Max concurrent request count should be 10", 10, AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT); - assertEquals("Max block size should be 100MB", 100L * 1024L * 1024L, AzureConstants.AZURE_BLOB_MAX_BLOCK_SIZE); - assertEquals("Max retry requests should be 4", 4, AzureConstants.AZURE_BLOB_MAX_RETRY_REQUESTS); - assertEquals("Default timeout should be 60 seconds", 60, AzureConstants.AZURE_BLOB_DEFAULT_TIMEOUT_SECONDS); + public void testV8AndV12MaxPartSizeDiffer() { + assertNotEquals("MAX_MULTIPART_UPLOAD_PART_SIZE must differ between V8 and V12", + AzureConstantsV8.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE, + AzureConstantsV12.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE); } @Test - public void testMetadataKeyPrefixConstruction() { - // Test that the metadata key prefix is correctly constructed - String expectedPrefix = AzureConstants.AZURE_BlOB_META_DIR_NAME + "/"; - assertEquals("Metadata key prefix should be META/", expectedPrefix, AzureConstants.AZURE_BLOB_META_KEY_PREFIX); + public void testV8AndV12MinPartSizeDiffer() { + assertNotEquals("MIN_MULTIPART_UPLOAD_PART_SIZE must differ between V8 and V12", + AzureConstantsV8.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE, + AzureConstantsV12.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE); } @Test - public void testConstantsAreNotNull() { - // Ensure no constants are null - assertNotNull("AZURE_STORAGE_ACCOUNT_NAME should not be null", AzureConstants.AZURE_STORAGE_ACCOUNT_NAME); - assertNotNull("AZURE_STORAGE_ACCOUNT_KEY should not be null", AzureConstants.AZURE_STORAGE_ACCOUNT_KEY); - assertNotNull("AZURE_CONNECTION_STRING should not be null", AzureConstants.AZURE_CONNECTION_STRING); - assertNotNull("AZURE_SAS should not be null", AzureConstants.AZURE_SAS); - assertNotNull("AZURE_TENANT_ID should not be null", AzureConstants.AZURE_TENANT_ID); - assertNotNull("AZURE_CLIENT_ID should not be null", AzureConstants.AZURE_CLIENT_ID); - assertNotNull("AZURE_CLIENT_SECRET should not be null", AzureConstants.AZURE_CLIENT_SECRET); - assertNotNull("AZURE_BLOB_ENDPOINT should not be null", AzureConstants.AZURE_BLOB_ENDPOINT); - assertNotNull("AZURE_BLOB_CONTAINER_NAME should not be null", AzureConstants.AZURE_BLOB_CONTAINER_NAME); - assertNotNull("AZURE_BlOB_META_DIR_NAME should not be null", AzureConstants.AZURE_BlOB_META_DIR_NAME); - assertNotNull("AZURE_BLOB_META_KEY_PREFIX should not be null", AzureConstants.AZURE_BLOB_META_KEY_PREFIX); - assertNotNull("AZURE_BLOB_REF_KEY should not be null", AzureConstants.AZURE_BLOB_REF_KEY); - assertNotNull("AZURE_BLOB_LAST_MODIFIED_KEY should not be null", AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY); + public void testV8AndV12BufferedStreamThresholdDiffer() { + assertNotEquals("BUFFERED_STREAM_THRESHOLD must differ between V8 and V12", + AzureConstantsV8.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD, + AzureConstantsV12.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD); } @Test - public void testConstantsAreNotEmpty() { - // Ensure no string constants are empty - assertFalse("AZURE_STORAGE_ACCOUNT_NAME should not be empty", AzureConstants.AZURE_STORAGE_ACCOUNT_NAME.isEmpty()); - assertFalse("AZURE_STORAGE_ACCOUNT_KEY should not be empty", AzureConstants.AZURE_STORAGE_ACCOUNT_KEY.isEmpty()); - assertFalse("AZURE_CONNECTION_STRING should not be empty", AzureConstants.AZURE_CONNECTION_STRING.isEmpty()); - assertFalse("AZURE_SAS should not be empty", AzureConstants.AZURE_SAS.isEmpty()); - assertFalse("AZURE_TENANT_ID should not be empty", AzureConstants.AZURE_TENANT_ID.isEmpty()); - assertFalse("AZURE_CLIENT_ID should not be empty", AzureConstants.AZURE_CLIENT_ID.isEmpty()); - assertFalse("AZURE_CLIENT_SECRET should not be empty", AzureConstants.AZURE_CLIENT_SECRET.isEmpty()); - assertFalse("AZURE_BLOB_ENDPOINT should not be empty", AzureConstants.AZURE_BLOB_ENDPOINT.isEmpty()); - assertFalse("AZURE_BLOB_CONTAINER_NAME should not be empty", AzureConstants.AZURE_BLOB_CONTAINER_NAME.isEmpty()); - assertFalse("AZURE_BlOB_META_DIR_NAME should not be empty", AzureConstants.AZURE_BlOB_META_DIR_NAME.isEmpty()); - assertFalse("AZURE_BLOB_META_KEY_PREFIX should not be empty", AzureConstants.AZURE_BLOB_META_KEY_PREFIX.isEmpty()); - assertFalse("AZURE_BLOB_REF_KEY should not be empty", AzureConstants.AZURE_BLOB_REF_KEY.isEmpty()); - assertFalse("AZURE_BLOB_LAST_MODIFIED_KEY should not be empty", AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY.isEmpty()); + public void testV8AndV12DefaultConcurrencyDiffer() { + assertNotEquals("DEFAULT_CONCURRENT_REQUEST_COUNT must differ between V8 and V12", + AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT, + AzureConstantsV12.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); } @Test - public void testSizeConstants() { - // Test that size constants are reasonable and in correct order - assertTrue("Min part size should be less than max part size", - AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE < AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE); - assertTrue("Max single PUT size should be less than max binary size", - AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE < AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE); - assertTrue("Buffered stream threshold should be reasonable", - AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD > 0 && AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD < 100L * 1024L * 1024L); - assertTrue("Max block size should be reasonable", - AzureConstants.AZURE_BLOB_MAX_BLOCK_SIZE > 0 && AzureConstants.AZURE_BLOB_MAX_BLOCK_SIZE <= 1024L * 1024L * 1024L); + public void testV8AndV12MaxConcurrencyDiffer() { + assertNotEquals("MAX_CONCURRENT_REQUEST_COUNT must differ between V8 and V12", + AzureConstantsV8.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT, + AzureConstantsV12.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT); } @Test - public void testConcurrencyConstants() { - // Test concurrency-related constants - assertTrue("Default concurrent request count should be positive", AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT > 0); - assertTrue("Max concurrent request count should be greater than or equal to default", - AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT >= AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); - assertTrue("Max allowable upload URIs should be positive", AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS > 0); - assertTrue("Max unique record tries should be positive", AzureConstants.AZURE_BLOB_MAX_UNIQUE_RECORD_TRIES > 0); - assertTrue("Max retry requests should be positive", AzureConstants.AZURE_BLOB_MAX_RETRY_REQUESTS > 0); - assertTrue("Default timeout should be positive", AzureConstants.AZURE_BLOB_DEFAULT_TIMEOUT_SECONDS > 0); + public void testV8AndV12MaxBinaryUploadSizeDiffer() { + assertNotEquals("MAX_BINARY_UPLOAD_SIZE must differ between V8 and V12", + AzureConstantsV8.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE, + AzureConstantsV12.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE); } } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java deleted file mode 100644 index 46b9379948b..00000000000 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderIT.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; - -import java.io.File; -import java.io.IOException; -import java.net.URI; - -import javax.net.ssl.HttpsURLConnection; - -import org.apache.commons.lang3.StringUtils; -import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; -import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; -import org.apache.jackrabbit.oak.spi.blob.data.DataStore; -import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.AbstractDataRecordAccessProviderIT; -import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.rules.TemporaryFolder; - -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStoreUtils.getAzureConfig; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStoreUtils.getAzureDataStore; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStoreUtils.isAzureConfigured; -import static org.junit.Assume.assumeTrue; - -/** - * As the test is memory intensive requires -Dtest.opts.memory=-Xmx2G - */ -public class AzureDataRecordAccessProviderIT extends AbstractDataRecordAccessProviderIT { - - @ClassRule - public static TemporaryFolder homeDir = new TemporaryFolder(new File("target")); - - private static AzureDataStore dataStore; - - @BeforeClass - public static void setupDataStore() throws Exception { - assumeTrue(isAzureConfigured() && !StringUtils.isEmpty(System.getProperty("test.opts.memory"))); - - dataStore = (AzureDataStore) getAzureDataStore(getAzureConfig(), homeDir.newFolder().getAbsolutePath()); - dataStore.setDirectDownloadURIExpirySeconds(expirySeconds); - dataStore.setDirectUploadURIExpirySeconds(expirySeconds); - } - - @Override - protected ConfigurableDataRecordAccessProvider getDataStore() { - return dataStore; - } - - @Override - protected DataRecord doGetRecord(DataStore ds, DataIdentifier identifier) throws DataStoreException { - return ds.getRecord(identifier); - } - - @Override - protected void doDeleteRecord(DataStore ds, DataIdentifier identifier) throws DataStoreException { - ((AzureDataStore)ds).deleteRecord(identifier); - } - - @Override - protected long getProviderMaxPartSize() { - return AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; - } - - @Override - protected HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException { - return AzureDataStoreUtils.getHttpsConnection(length, uri); - } -} diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreIT.java similarity index 85% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreIT.java index 2f42b527d62..f6752037c23 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreIT.java @@ -18,10 +18,46 @@ import static org.apache.commons.codec.binary.Hex.encodeHexString; import static org.apache.commons.io.FileUtils.copyInputStreamToFile; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.net.URI; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; -import com.azure.storage.blob.BlobContainerClient; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.NullOutputStream; @@ -29,6 +65,7 @@ import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8; import org.apache.jackrabbit.oak.commons.collections.IteratorUtils; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions; @@ -36,32 +73,38 @@ import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions; import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend; import org.apache.jackrabbit.oak.spi.blob.SharedBackend; -import org.junit.*; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.MockitoAnnotations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; -import javax.jcr.RepositoryException; -import java.io.*; -import java.net.URI; -import java.security.DigestOutputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.*; - /** * Combined unit and integration tests for AzureDataStore class. * Unit tests use Mockito and don't require Azure configuration. * Integration tests use Azurite (Azure Storage emulator) for local testing. + * + *

Parameterized over {@link AzureSdkVersion} so that every integration test + * runs against both SDK v8 and SDK v12. */ -@RunWith(MockitoJUnitRunner.class) -public class AzureDataStoreTest { - protected static final Logger LOG = LoggerFactory.getLogger(AzureDataStoreTest.class); +@RunWith(Parameterized.class) +public class AzureDataStoreIT { + protected static final Logger LOG = LoggerFactory.getLogger(AzureDataStoreIT.class); + + @Parameterized.Parameters(name = "SDK {0}") + public static Object[] sdkVersions() { + return AzureSdkVersion.values(); + } + + @Parameterized.Parameter + public AzureSdkVersion sdkVersion; // Azurite Docker container for integration tests @ClassRule @@ -82,14 +125,24 @@ public class AzureDataStoreTest { private AzureDataStore ds; private AbstractAzureBlobStoreBackend backend; private String container; - private BlobContainerClient azuriteContainer; + private AzureBlobContainer azuriteContainer; Random randomGen = new Random(); + private AutoCloseable mocks; + @Before public void setUp() { + mocks = MockitoAnnotations.openMocks(this); azureDataStore = new AzureDataStore(); } + @After + public void closeMocks() throws Exception { + if (mocks != null) { + mocks.close(); + } + } + //// // Unit Tests - Testing AzureDataStore logic without Azure backend //// @@ -117,7 +170,7 @@ public void testSetPropertiesWithNull() { azureDataStore.setProperties(null); // Should not throw exception - assertTrue("Should not throw exception", true); + } @Test @@ -174,7 +227,7 @@ public void testSetBinaryTransferAccelerationEnabled() { azureDataStore.setBinaryTransferAccelerationEnabled(true); azureDataStore.setBinaryTransferAccelerationEnabled(false); // No exception should be thrown - assertTrue("Should not throw exception", true); + } @Test @@ -184,7 +237,7 @@ public void testSetDirectUploadURIExpirySecondsWithoutBackend() { azureDataStore.setDirectUploadURIExpirySeconds(0); azureDataStore.setDirectUploadURIExpirySeconds(-1); // No exception should be thrown - assertTrue("Should not throw exception", true); + } @Test @@ -195,7 +248,7 @@ public void testSetDirectDownloadURIExpirySecondsWithoutBackend() { azureDataStore.setDirectDownloadURIExpirySeconds(-1); // No exception should be thrown - assertTrue("Should not throw exception", true); + } @Test(expected = DataRecordUploadException.class) @@ -222,14 +275,7 @@ public void testGetDownloadURIWithoutBackend() { @Test public void testSetDirectDownloadURICacheSizeWithoutBackend() { - // This should call the method on null backend, which will cause NPE - // But looking at the implementation, it doesn't check for null like the other methods - try { - azureDataStore.setDirectDownloadURICacheSize(100); - fail("Expected NullPointerException"); - } catch (NullPointerException e) { - // Expected behavior since the method doesn't check for null backend - } + azureDataStore.setDirectDownloadURICacheSize(100); } @Test @@ -278,7 +324,7 @@ public void testSetDirectUploadURIExpirySecondsWithBackend() { azureDataStore.setDirectUploadURIExpirySeconds(-1); // Should not throw exceptions - assertTrue("Should not throw exception", true); + } @Test @@ -292,7 +338,7 @@ public void testSetDirectDownloadURIExpirySecondsWithBackend() { azureDataStore.setDirectDownloadURIExpirySeconds(-1); // Should not throw exceptions - assertTrue("Should not throw exception", true); + } @Test @@ -306,7 +352,7 @@ public void testSetDirectDownloadURICacheSizeWithBackend() { azureDataStore.setDirectDownloadURICacheSize(-1); // Should not throw exceptions - assertTrue("Should not throw exception", true); + } @Test(expected = NullPointerException.class) @@ -407,7 +453,7 @@ public void testBackendInstantiationWithAzureSdk12Enabled() { // Verify that the backend is an instance of AzureBlobStoreBackend (SDK 12) assertNotNull("Backend should not be null", backend); assertTrue("Backend should be an instance of AzureBlobStoreBackend when SDK 12 is enabled", - backend instanceof AzureBlobStoreBackend); + backend instanceof AzureBlobStoreBackendV12); assertFalse("Backend should not be an instance of AzureBlobStoreBackendV8 when SDK 12 is enabled", backend instanceof AzureBlobStoreBackendV8); } finally { @@ -420,6 +466,29 @@ public void testBackendInstantiationWithAzureSdk12Enabled() { } } + @Test + public void testBackendPropertiesOverrideSystemProperty() { + String originalProperty = System.getProperty("blob.azure.v12.enabled"); + try { + System.setProperty("blob.azure.v12.enabled", "false"); + Properties properties = new Properties(); + properties.setProperty(AzureConstants.AZURE_V12_ENABLED_PROPERTY, "true"); + + AzureDataStore dataStore = new AzureDataStore(); + dataStore.setProperties(properties); + + AbstractSharedBackend backend = dataStore.createBackend(); + + assertTrue(backend instanceof AzureBlobStoreBackendV12); + } finally { + if (originalProperty != null) { + System.setProperty("blob.azure.v12.enabled", originalProperty); + } else { + System.clearProperty("blob.azure.v12.enabled"); + } + } + } + @Test public void testBackendInstantiationWithAzureSdk12Disabled() { String originalProperty = System.getProperty("blob.azure.v12.enabled"); @@ -483,7 +552,7 @@ public void testConfigurableDataRecordAccessProviderMethods() { azureDataStore.setBinaryTransferAccelerationEnabled(false); // These should not throw exceptions even without backend - assertTrue("Should not throw exception", true); + } @Test @@ -549,7 +618,7 @@ public void testCreateBackendSetsAzureBlobStoreBackendField() { azureDataStore.setDirectDownloadURICacheSize(100); // No exceptions should be thrown - assertTrue("Should not throw exception", true); + } //// @@ -557,17 +626,18 @@ public void testCreateBackendSetsAzureBlobStoreBackendField() { // These tests use Azurite Docker container for local testing without requiring real Azure credentials //// - private void setupIntegrationTest() throws IOException, RepositoryException { + private void setupIntegrationTest() throws Exception { // Generate unique container name for this test run container = String.valueOf(randomGen.nextInt(9999)) + "-" + String.valueOf(randomGen.nextInt(9999)) + "-test"; - // Create Azurite container - azuriteContainer = azurite.getContainer(container, getConnectionString()); - // Setup properties for Azurite props = createAzuriteProperties(); + // Create Azurite container via SDK-neutral factory + AzureBlobContainers.deleteIfExists(props); + azuriteContainer = AzureBlobContainers.create(props); + ds = new AzureDataStore(); ds.setProperties(props); ds.setCacheSize(0); // Turn caching off so we don't get weird test results due to caching @@ -598,7 +668,8 @@ private void teardownIntegrationTest() { } /** - * Creates properties configured for Azurite local testing. + * Creates properties configured for Azurite local testing, + * selecting the SDK version supplied by the parameterized runner. */ private Properties createAzuriteProperties() { Properties properties = new Properties(); @@ -606,8 +677,10 @@ private Properties createAzuriteProperties() { properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); - properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "true"); - properties.setProperty(AzureConstants.AZURE_REF_ON_INIT, "true"); // Enable reference key creation + properties.setProperty(AzureConstants.AZURE_V12_ENABLED_PROPERTY, + String.valueOf(sdkVersion == AzureSdkVersion.V12)); + properties.setProperty("azureCreateContainer", "true"); + properties.setProperty("refOnInit", "true"); // Enable reference key creation return properties; } @@ -615,11 +688,7 @@ private Properties createAzuriteProperties() { * Gets the Azurite connection string. */ private static String getConnectionString() { - return Utils.getConnectionString( - AzuriteDockerRule.ACCOUNT_NAME, - AzuriteDockerRule.ACCOUNT_KEY, - azurite.getBlobEndpoint() - ); + return azurite.getConnectionString(); } private void validateRecord(final DataRecord record, @@ -790,41 +859,8 @@ public void testBackendWriteDifferentSizedRecords() throws Exception { } } - @Test - public void testBackendWriteRecordNullIdentifierThrowsNullPointerException() throws Exception { - setupIntegrationTest(); - try { - DataIdentifier identifier = null; - File testFile = folder.newFile(); - copyInputStreamToFile(randomStream(0, 10), testFile); - try { - backend.write(identifier, testFile); - fail(); - } catch (NullPointerException e) { - assertEquals("identifier must not be null", e.getMessage()); - } - } finally { - teardownIntegrationTest(); - } - } - - @Test - public void testBackendWriteRecordNullFileThrowsNullPointerException() throws Exception { - setupIntegrationTest(); - try { - File testFile = null; - DataIdentifier identifier = new DataIdentifier("fake"); - try { - backend.write(identifier, testFile); - fail(); - } - catch (NullPointerException e) { - assertTrue("file must not be null".equals(e.getMessage())); - } - } finally { - teardownIntegrationTest(); - } - } + // Version-specific null parameter validation tests (with message checks) are in + // v8/AzureBlobStoreBackendV8IT and v12/AzureBlobStoreBackendV12IT @Test public void testBackendWriteRecordFileNotFoundThrowsException() throws Exception { @@ -845,22 +881,6 @@ public void testBackendWriteRecordFileNotFoundThrowsException() throws Exception } } - @Test - public void testBackendReadRecordNullIdentifier() throws Exception { - setupIntegrationTest(); - try { - DataIdentifier identifier = null; - try { - backend.read(identifier); - fail(); - } - catch (NullPointerException e) { - assert("identifier must not be null".equals(e.getMessage())); - } - } finally { - teardownIntegrationTest(); - } - } @Test(expected = DataStoreException.class) public void testBackendReadRecordInvalidIdentifier() throws Exception { @@ -880,22 +900,6 @@ public void testBackendReadRecordInvalidIdentifier() throws Exception { } } - @Test - public void testBackendDeleteRecordNullIdentifier() throws Exception { - setupIntegrationTest(); - try { - DataIdentifier identifier = null; - try { - backend.deleteRecord(identifier); - fail(); - } - catch (NullPointerException e) { - assert("identifier must not be null".equals(e.getMessage())); - } - } finally { - teardownIntegrationTest(); - } - } @Test public void testBackendDeleteRecordInvalidIdentifier() throws Exception { @@ -984,22 +988,6 @@ public void testBackendGetRecord() throws Exception { } } - @Test - public void testBackendGetRecordNullIdentifierThrowsNullPointerException() throws Exception { - setupIntegrationTest(); - try { - try { - DataIdentifier identifier = null; - backend.getRecord(identifier); - fail(); - } - catch (NullPointerException e) { - assertTrue("identifier must not be null".equals(e.getMessage())); - } - } finally { - teardownIntegrationTest(); - } - } @Test public void testBackendGetRecordInvalidIdentifierThrowsDataStoreException() throws Exception { @@ -1115,63 +1103,6 @@ public void testBackendAddMetadataRecordFileNotFoundThrowsDataStoreException() t } } - @Test - public void testBackendAddMetadataRecordNullInputStreamThrowsNullPointerException() throws Exception { - setupIntegrationTest(); - try { - try { - backend.addMetadataRecord((InputStream)null, "name"); - fail(); - } - catch (NullPointerException e) { - assertTrue("input must not be null".equals(e.getMessage())); - } - } finally { - teardownIntegrationTest(); - } - } - - @Test - public void testBackendAddMetadataRecordNullFileThrowsNullPointerException() throws Exception { - setupIntegrationTest(); - try { - try { - backend.addMetadataRecord((File)null, "name"); - fail(); - } - catch (NullPointerException e) { - assertTrue("input must not be null".equals(e.getMessage())); - } - } finally { - teardownIntegrationTest(); - } - } - - @Test - public void testBackendAddMetadataRecordNullEmptyNameThrowsIllegalArgumentException() throws Exception { - setupIntegrationTest(); - try { - final String data = "testData"; - for (boolean fromInputStream : List.of(false, true)) { - for (String name : Arrays.asList(null, "")) { - try { - if (fromInputStream) { - backend.addMetadataRecord(new ByteArrayInputStream(data.getBytes()), name); - } else { - File testFile = folder.newFile(); - copyInputStreamToFile(new ByteArrayInputStream(data.getBytes()), testFile); - backend.addMetadataRecord(testFile, name); - } - fail(); - } catch (IllegalArgumentException e) { - assertTrue("name should not be empty".equals(e.getMessage())); - } - } - } - } finally { - teardownIntegrationTest(); - } - } @Test public void testBackendGetMetadataRecordInvalidName() throws Exception { @@ -1224,21 +1155,6 @@ public void testBackendGetAllMetadataRecordsPrefixMatchesAll() throws Exception } } - @Test - public void testBackendGetAllMetadataRecordsNullPrefixThrowsNullPointerException() throws Exception { - setupIntegrationTest(); - try { - try { - backend.getAllMetadataRecords(null); - fail(); - } - catch (NullPointerException e) { - assertEquals("prefix must not be null", e.getMessage()); - } - } finally { - teardownIntegrationTest(); - } - } @Test public void testBackendDeleteMetadataRecord() throws Exception { @@ -1334,21 +1250,6 @@ public void testBackendDeleteAllMetadataRecordsNoRecordsNoChange() throws Except } } - @Test - public void testBackendDeleteAllMetadataRecordsNullPrefixThrowsNullPointerException() throws Exception { - setupIntegrationTest(); - try { - try { - backend.deleteAllMetadataRecords(null); - fail(); - } - catch (NullPointerException e) { - assertEquals("prefix must not be null", e.getMessage()); - } - } finally { - teardownIntegrationTest(); - } - } @Test public void testSecret() throws Exception { diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java index 640fb1f394c..a49488c0814 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreUtils.java @@ -18,73 +18,44 @@ */ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; -import static org.junit.Assume.assumeTrue; - import java.io.File; import java.io.FileInputStream; -import java.io.IOException; import java.io.InputStream; -import java.net.URI; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.Map; import java.util.Properties; -import javax.net.ssl.HttpsURLConnection; - -import com.azure.storage.blob.BlobContainerClient; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataStore; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.UtilsV12; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.UtilsV8; import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.commons.collections.MapUtils; import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils; -import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Extension to {@link DataStoreUtils} to enable Azure extensions for cleaning and initialization. + * Test utility for creating and managing Azure DataStore instances. + * Used by oak-run, oak-it and other modules for integration testing with real Azure credentials. */ public class AzureDataStoreUtils extends DataStoreUtils { + private static final Logger log = LoggerFactory.getLogger(AzureDataStoreUtils.class); private static final String DEFAULT_CONFIG_PATH = "./src/test/resources/azure.properties"; private static final String DEFAULT_PROPERTY_FILE = "azure.properties"; private static final String SYS_PROP_NAME = "azure.config"; - /** - * Check for presence of mandatory properties. - * - * @return true if mandatory props configured. - */ public static boolean isAzureConfigured() { Properties props = getAzureConfig(); - //need either access keys or sas or service principal - if (!props.containsKey(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY) || !props.containsKey(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME) - || !(props.containsKey(AzureConstants.AZURE_BLOB_CONTAINER_NAME))) { - if (!props.containsKey(AzureConstants.AZURE_SAS) || !props.containsKey(AzureConstants.AZURE_BLOB_ENDPOINT) - || !(props.containsKey(AzureConstants.AZURE_BLOB_CONTAINER_NAME))) { - // service principal - return props.containsKey(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME) && props.containsKey(AzureConstants.AZURE_TENANT_ID) && - props.containsKey(AzureConstants.AZURE_CLIENT_ID) && props.containsKey(AzureConstants.AZURE_CLIENT_SECRET) && - props.containsKey(AzureConstants.AZURE_BLOB_CONTAINER_NAME); - } + AzureSdkVersion version = AzureSdkVersion.resolve(props); + if (version == AzureSdkVersion.V12) { + return UtilsV12.isConfigured(props); } - return true; + return UtilsV8.isConfigured(props); } - /** - * Read any config property configured. - * Also, read any props available as system properties. - * System properties take precedence. - * - * @return Properties instance - */ public static Properties getAzureConfig() { String config = System.getProperty(SYS_PROP_NAME); if (StringUtils.isEmpty(config)) { @@ -108,14 +79,13 @@ public static Properties getAzureConfig() { } finally { IOUtils.closeQuietly(is); } - props.putAll(getConfig()); + props.putAll(DataStoreUtils.getConfig()); Map filtered = MapUtils.filterEntries(MapUtils.fromProperties(props), input -> !StringUtils.isEmpty(input.getValue())); props = new Properties(); props.putAll(filtered); } - props.setProperty("blob.azure.v12.enabled", "true"); return props; } @@ -124,51 +94,9 @@ public static DataStore getAzureDataStore(Properties props, String homeDir) thro PropertiesUtil.populate(ds, MapUtils.fromProperties(props), false); ds.setProperties(props); ds.init(homeDir); - - return ds; - } - - public static T setupDirectAccessDataStore( - @NotNull final TemporaryFolder homeDir, - int directDownloadExpirySeconds, - int directUploadExpirySeconds) - throws Exception { - return setupDirectAccessDataStore(homeDir, directDownloadExpirySeconds, directUploadExpirySeconds, null); - } - - @SuppressWarnings("unchecked") - public static T setupDirectAccessDataStore( - @NotNull final TemporaryFolder homeDir, - int directDownloadExpirySeconds, - int directUploadExpirySeconds, - @Nullable final Properties overrideProperties) - throws Exception { - assumeTrue(isAzureConfigured()); - T ds = (T) getAzureDataStore(getDirectAccessDataStoreProperties(overrideProperties), homeDir.newFolder().getAbsolutePath()); - if (ds instanceof ConfigurableDataRecordAccessProvider) { - ((ConfigurableDataRecordAccessProvider) ds).setDirectDownloadURIExpirySeconds(directDownloadExpirySeconds); - ((ConfigurableDataRecordAccessProvider) ds).setDirectUploadURIExpirySeconds(directUploadExpirySeconds); - } return ds; } - public static Properties getDirectAccessDataStoreProperties() { - return getDirectAccessDataStoreProperties(null); - } - - public static Properties getDirectAccessDataStoreProperties(@Nullable final Properties overrideProperties) { - Properties mergedProperties = new Properties(); - mergedProperties.putAll(getAzureConfig()); - if (overrideProperties != null) { - mergedProperties.putAll(overrideProperties); - } - // set properties needed for direct access testing - if (mergedProperties.getProperty("cacheSize", null) == null) { - mergedProperties.put("cacheSize", "0"); - } - return mergedProperties; - } - public static void deleteContainer(String containerName) throws Exception { if (StringUtils.isEmpty(containerName)) { log.warn("Cannot delete container with null or empty name. containerName={}", containerName); @@ -178,22 +106,7 @@ public static void deleteContainer(String containerName) throws Exception { Properties props = getAzureConfig(); props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, containerName); - AzureBlobContainerProvider azureBlobContainerProvider = AzureBlobContainerProvider.Builder.builder(containerName).initializeWithProperties(props).build(); - BlobContainerClient container = azureBlobContainerProvider.getBlobContainer(); - boolean result = container.deleteIfExists(); + boolean result = AzureBlobContainers.deleteIfExists(props); log.info("Container deleted. containerName={} existed={}", containerName, result); } - - protected static HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException { - HttpsURLConnection conn = (HttpsURLConnection) uri.toURL().openConnection(); - conn.setDoOutput(true); - conn.setRequestMethod("PUT"); - conn.setRequestProperty("Content-Length", String.valueOf(length)); - conn.setRequestProperty("Date", DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssX") - .withZone(ZoneOffset.UTC) - .format(Instant.now())); - conn.setRequestProperty("x-ms-version", "2017-11-09"); - - return conn; - } } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreVersionSelectionIT.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreVersionSelectionIT.java new file mode 100644 index 00000000000..179e7553a47 --- /dev/null +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreVersionSelectionIT.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; + +import java.io.File; +import java.util.List; +import java.util.Properties; +import java.util.UUID; +import java.util.stream.Collectors; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; + +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureBlobStoreBackendV12; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertTrue; + +public class AzureDataStoreVersionSelectionIT { + + private static final String AZURE_SDK_12_ENABLED = "blob.azure.v12.enabled"; + + @ClassRule + public static AzuriteDockerRule azurite = new AzuriteDockerRule(); + + @Rule + public TemporaryFolder folder = new TemporaryFolder(new File("target")); + + private final String originalProperty = System.getProperty(AZURE_SDK_12_ENABLED); + + private AzureDataStore dataStore; + private AzureBlobContainer azuriteContainer; + + @After + public void tearDown() throws Exception { + restoreProperty(); + if (dataStore != null) { + dataStore.close(); + dataStore = null; + } + if (azuriteContainer != null) { + azuriteContainer.deleteIfExists(); + azuriteContainer = null; + } + } + + @Test + public void logsSdk12WhenPropertyEnabled() throws Exception { + assertStartupLog("true", null, "Starting blob store using Azure SDK 12", AzureBlobStoreBackendV12.class); + } + + @Test + public void logsSdk8WhenPropertyDisabled() throws Exception { + assertStartupLog("false", null, "Starting blob store using Azure SDK 8", AzureBlobStoreBackendV8.class); + } + + @Test + public void logsSdk8WhenPropertyUnset() throws Exception { + assertStartupLog(null, null, "Starting blob store using Azure SDK 8", AzureBlobStoreBackendV8.class); + } + + @Test + public void propertiesOverrideSystemPropertyWhenEnablingSdk12() throws Exception { + assertStartupLog("false", "true", "Starting blob store using Azure SDK 12", AzureBlobStoreBackendV12.class); + } + + @Test + public void propertiesOverrideSystemPropertyWhenDisablingSdk12() throws Exception { + assertStartupLog("true", "false", "Starting blob store using Azure SDK 8", AzureBlobStoreBackendV8.class); + } + + private void assertStartupLog(String systemPropertyValue, String configuredValue, String expectedMessage, Class backendType) throws Exception { + if (systemPropertyValue == null) { + System.clearProperty(AZURE_SDK_12_ENABLED); + } else { + System.setProperty(AZURE_SDK_12_ENABLED, systemPropertyValue); + } + + ListAppender appender = subscribeAppender(); + try { + String containerName = "it-" + UUID.randomUUID(); + Properties props = createAzuriteProperties(containerName, configuredValue); + azuriteContainer = AzureBlobContainers.create(props); + + dataStore = new AzureDataStore(); + dataStore.setProperties(props); + dataStore.setCacheSize(0); + dataStore.init(folder.newFolder().getAbsolutePath()); + + assertTrue(backendType.isInstance(dataStore.getBackend())); + assertTrue(getMessages(appender).contains(expectedMessage)); + } finally { + unsubscribe(appender); + } + } + + private static String getConnectionString() { + return azurite.getConnectionString(); + } + + private Properties createAzuriteProperties(String containerName, String configuredValue) { + Properties properties = new Properties(); + properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, containerName); + properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); + properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); + properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); + if (configuredValue != null) { + properties.setProperty(AzureConstants.AZURE_V12_ENABLED_PROPERTY, configuredValue); + } + properties.setProperty("azureCreateContainer", "true"); + properties.setProperty("refOnInit", "true"); + return properties; + } + + private static List getMessages(ListAppender appender) { + return appender.list.stream().map(ILoggingEvent::getFormattedMessage).collect(Collectors.toList()); + } + + private static ListAppender subscribeAppender() { + ListAppender appender = new ListAppender<>(); + appender.start(); + ((Logger) LoggerFactory.getLogger(AzureDataStore.class)).addAppender(appender); + return appender; + } + + private static void unsubscribe(ListAppender appender) { + ((Logger) LoggerFactory.getLogger(AzureDataStore.class)).detachAppender(appender); + appender.stop(); + } + + private void restoreProperty() { + if (originalProperty == null) { + System.clearProperty(AZURE_SDK_12_ENABLED); + } else { + System.setProperty(AZURE_SDK_12_ENABLED, originalProperty); + } + } +} diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureIsolationTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureIsolationTest.java new file mode 100644 index 00000000000..c6043bbbb98 --- /dev/null +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureIsolationTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertTrue; + +/** + * Source-level isolation test that scans all .java files to enforce that: + *

    + *
  • v8 package has no V12 storage SDK imports and no v12 cross-package references
    • + *
  • v12 package has no V8 SDK imports and no v8 cross-package references
    • + *
  • Parent blobstorage package has no Azure SDK storage imports
    • + *
+ * + * Exception: {@code com.azure.identity.*} and {@code com.azure.core.credential.*} are + * allowed in v8 for AAD service principal authentication (see OAK-12164 plan section 2.4). + */ +public class AzureIsolationTest { + + // V12 storage SDK pattern (forbidden in v8 and parent) + private static final Pattern V12_STORAGE_SDK = Pattern.compile( + "import\\s+com\\.azure\\.storage\\."); + + // V8 SDK pattern (forbidden in v12 and parent) + private static final Pattern V8_SDK = Pattern.compile( + "import\\s+com\\.microsoft\\.azure\\."); + + // Cross-package references + private static final Pattern V12_PACKAGE_REF = Pattern.compile( + "blobstorage\\.v12\\."); + + private static final Pattern V8_PACKAGE_REF = Pattern.compile( + "blobstorage\\.v8\\."); + + private static final Path SRC_ROOT = findSourceRoot("src/main/java"); + private static final Path TEST_ROOT = findSourceRoot("src/test/java"); + + private static final String BLOBSTORAGE_PKG = "org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage"; + + @Test + public void v8SourceMustNotImportV12StorageSDK() throws IOException { + List violations = new ArrayList<>(); + scanPackage(SRC_ROOT, "v8", V12_STORAGE_SDK, "com.azure.storage.* import", violations); + scanPackage(TEST_ROOT, "v8", V12_STORAGE_SDK, "com.azure.storage.* import", violations); + assertNoViolations("v8 must not import V12 storage SDK", violations); + } + + @Test + public void v8SourceMustNotReferenceV12Package() throws IOException { + List violations = new ArrayList<>(); + scanPackage(SRC_ROOT, "v8", V12_PACKAGE_REF, "blobstorage.v12.* reference", violations); + scanPackage(TEST_ROOT, "v8", V12_PACKAGE_REF, "blobstorage.v12.* reference", violations); + assertNoViolations("v8 must not reference v12 package", violations); + } + + @Test + public void v12SourceMustNotImportV8SDK() throws IOException { + List violations = new ArrayList<>(); + scanPackage(SRC_ROOT, "v12", V8_SDK, "com.microsoft.azure.* import", violations); + scanPackage(TEST_ROOT, "v12", V8_SDK, "com.microsoft.azure.* import", violations); + assertNoViolations("v12 must not import V8 SDK", violations); + } + + @Test + public void v12SourceMustNotReferenceV8Package() throws IOException { + List violations = new ArrayList<>(); + scanPackage(SRC_ROOT, "v12", V8_PACKAGE_REF, "blobstorage.v8.* reference", violations); + scanPackage(TEST_ROOT, "v12", V8_PACKAGE_REF, "blobstorage.v8.* reference", violations); + assertNoViolations("v12 must not reference v8 package", violations); + } + + @Test + public void parentSourcePackageMustNotImportAzureStorageSDK() throws IOException { + // Only check src/main/java — test infrastructure (AzuriteDockerRule, etc.) + // legitimately needs both SDK types to create containers for both backends. + List violations = new ArrayList<>(); + scanParentPackage(SRC_ROOT, V12_STORAGE_SDK, "com.azure.storage.* import", violations); + scanParentPackage(SRC_ROOT, V8_SDK, "com.microsoft.azure.* import", violations); + assertNoViolations("parent blobstorage source package must not import Azure SDK storage types", violations); + } + + /** + * Scan Java files in a versioned subpackage (v8/ or v12/) for forbidden patterns. + */ + private void scanPackage(Path sourceRoot, String subpackage, Pattern forbidden, + String description, List violations) throws IOException { + Path pkgDir = sourceRoot.resolve(BLOBSTORAGE_PKG).resolve(subpackage); + if (!Files.isDirectory(pkgDir)) { + return; + } + Files.walkFileTree(pkgDir, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.toString().endsWith(".java")) { + checkFile(file, forbidden, description, violations); + } + return FileVisitResult.CONTINUE; + } + }); + } + + /** + * Scan Java files in the parent blobstorage package ONLY (not subpackages). + */ + private void scanParentPackage(Path sourceRoot, Pattern forbidden, + String description, List violations) throws IOException { + Path pkgDir = sourceRoot.resolve(BLOBSTORAGE_PKG); + if (!Files.isDirectory(pkgDir)) { + return; + } + // Only direct children, not v8/ or v12/ subdirectories + Files.list(pkgDir) + .filter(p -> p.toString().endsWith(".java")) + .filter(Files::isRegularFile) + .forEach(file -> { + try { + checkFile(file, forbidden, description, violations); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private void checkFile(Path file, Pattern forbidden, String description, + List violations) throws IOException { + List lines = Files.readAllLines(file); + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (forbidden.matcher(line).find()) { + // Check allowlist: com.azure.identity.* and com.azure.core.credential.* in v8 for AAD + if (isAllowedV8AadImport(file, line)) { + continue; + } + violations.add(String.format("%s:%d — %s: %s", + file.getFileName(), i + 1, description, line.trim())); + } + } + } + + /** + * com.azure.identity.* and com.azure.core.credential.* are allowed in v8 + * for AAD service principal authentication (the Azure Identity library is + * SDK-version-neutral). + */ + private boolean isAllowedV8AadImport(Path file, String line) { + if (!file.toString().contains("/v8/")) { + return false; + } + return line.contains("com.azure.identity.") || line.contains("com.azure.core.credential."); + } + + private void assertNoViolations(String context, List violations) { + assertTrue(context + ":\n" + String.join("\n", violations), violations.isEmpty()); + } + + private static Path findSourceRoot(String relativePath) { + // Walk up from this class's location to find the module root + Path moduleRoot = Paths.get("oak-blob-cloud-azure"); + if (!Files.isDirectory(moduleRoot)) { + // Try from current directory (might already be in module) + moduleRoot = Paths.get("."); + } + return moduleRoot.resolve(relativePath); + } +} diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java index 47cec3c1490..ea3b77d1855 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java @@ -23,6 +23,7 @@ import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; + import org.junit.Assume; import org.junit.rules.ExternalResource; import org.junit.runner.Description; @@ -37,6 +38,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; public class AzuriteDockerRule extends ExternalResource { @@ -103,6 +105,19 @@ public String getBlobEndpoint() { return "http://127.0.0.1:" + getMappedPort() + "/devstoreaccount1"; } + public String getConnectionString() { + return "DefaultEndpointsProtocol=http;AccountName=" + ACCOUNT_NAME + + ";AccountKey=" + ACCOUNT_KEY + + ";BlobEndpoint=" + getBlobEndpoint(); + } + + public Properties getProperties(String containerName) { + Properties properties = new Properties(); + properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); + properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, containerName); + return properties; + } + public CloudBlobContainer getContainer(String name) throws URISyntaxException, StorageException, InvalidKeyException { CloudStorageAccount cloud = getCloudStorageAccount(); CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient(); @@ -123,6 +138,11 @@ public BlobContainerClient getContainer(String containerName, String connectionS return blobContainerClient; } + public AzureBlobContainer getAzureBlobContainer(String containerName) throws Exception { + AzureBlobContainers.deleteIfExists(getProperties(containerName)); + return AzureBlobContainers.create(getProperties(containerName)); + } + public CloudStorageAccount getCloudStorageAccount() throws URISyntaxException, InvalidKeyException { String blobEndpoint = "BlobEndpoint=" + getBlobEndpoint(); String accountName = "AccountName=" + ACCOUNT_NAME; diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProviderTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerProviderV12IT.java similarity index 86% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProviderTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerProviderV12IT.java index e5d40a164a3..06c873a33ec 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProviderTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobContainerProviderV12IT.java @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import com.azure.identity.ClientSecretCredential; import com.azure.storage.blob.BlobContainerClient; @@ -39,19 +41,19 @@ import static org.junit.Assert.*; import static org.mockito.Mockito.*; -public class AzureBlobContainerProviderTest { +public class AzureBlobContainerProviderV12IT { @ClassRule public static AzuriteDockerRule azurite = new AzuriteDockerRule(); private static final String CONTAINER_NAME = "test-container"; - private AzureBlobContainerProvider provider; + private AzureBlobContainerProviderV12 provider; @Test public void testBuilderWithConnectionString() { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -63,7 +65,7 @@ public void testBuilderWithConnectionString() { @Test public void testBuilderWithAccountNameAndKey() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withAccountKey("testkey") @@ -75,7 +77,7 @@ public void testBuilderWithAccountNameAndKey() { @Test public void testBuilderWithServicePrincipal() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -89,7 +91,7 @@ public void testBuilderWithServicePrincipal() { @Test public void testBuilderWithSasToken() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withSasToken("sas-token") @@ -103,16 +105,16 @@ public void testBuilderWithSasToken() { @Test public void testBuilderInitializeWithProperties() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "testaccount"); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "testkey"); - properties.setProperty(AzureConstants.AZURE_TENANT_ID, "tenant-id"); - properties.setProperty(AzureConstants.AZURE_CLIENT_ID, "client-id"); - properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, "client-secret"); - properties.setProperty(AzureConstants.AZURE_SAS, "sas-token"); - properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "https://testaccount.blob.core.windows.net"); + properties.setProperty(AzureConstantsV12.AZURE_CONNECTION_STRING, getConnectionString()); + properties.setProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, "testaccount"); + properties.setProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, "testkey"); + properties.setProperty(AzureConstantsV12.AZURE_TENANT_ID, "tenant-id"); + properties.setProperty(AzureConstantsV12.AZURE_CLIENT_ID, "client-id"); + properties.setProperty(AzureConstantsV12.AZURE_CLIENT_SECRET, "client-secret"); + properties.setProperty(AzureConstantsV12.AZURE_SAS, "sas-token"); + properties.setProperty(AzureConstantsV12.AZURE_BLOB_ENDPOINT, "https://testaccount.blob.core.windows.net"); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .initializeWithProperties(properties) .build(); @@ -126,7 +128,7 @@ public void testBuilderInitializeWithProperties() { public void testGetBlobContainerWithConnectionString() throws DataStoreException { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -142,7 +144,7 @@ public void testGetBlobContainerWithRetryOptions() throws DataStoreException { RequestRetryOptions retryOptions = new RequestRetryOptions(); Properties properties = new Properties(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -155,14 +157,14 @@ public void testGetBlobContainerWithRetryOptions() throws DataStoreException { @Test public void testBuilderWithNullContainerName() { // Builder accepts null container name - no validation - AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder(null); + AzureBlobContainerProviderV12.Builder builder = AzureBlobContainerProviderV12.Builder.builder(null); assertNotNull("Builder should not be null", builder); } @Test public void testBuilderWithEmptyContainerName() { // Builder accepts empty container name - no validation - AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder(""); + AzureBlobContainerProviderV12.Builder builder = AzureBlobContainerProviderV12.Builder.builder(""); assertNotNull("Builder should not be null", builder); } @@ -170,7 +172,7 @@ public void testBuilderWithEmptyContainerName() { public void testGenerateSharedAccessSignatureWithConnectionString() { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -196,7 +198,7 @@ public void testGenerateSharedAccessSignatureWithConnectionString() { @Test public void testGetBlobContainerWithInvalidConnectionString() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString("invalid-connection-string") .build(); @@ -214,7 +216,7 @@ public void testGetBlobContainerWithInvalidConnectionString() { @Test public void testGetBlobContainerWithServicePrincipalMissingCredentials() throws DataStoreException{ - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -229,7 +231,7 @@ public void testGetBlobContainerWithServicePrincipalMissingCredentials() throws @Test(expected = DataStoreException.class) public void testGetBlobContainerWithSasTokenMissingEndpoint() throws DataStoreException{ - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withSasToken("sas-token") @@ -241,7 +243,7 @@ public void testGetBlobContainerWithSasTokenMissingEndpoint() throws DataStoreEx @Test public void testBuilderChaining() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString("connection1") .withAccountName("account1") @@ -260,7 +262,7 @@ public void testBuilderChaining() { @Test public void testBuilderWithEmptyStrings() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString("") .withAccountName("") @@ -280,11 +282,11 @@ public void testBuilderWithEmptyStrings() { @Test public void testInitializeWithPropertiesEmptyValues() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, ""); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, ""); + properties.setProperty(AzureConstantsV12.AZURE_CONNECTION_STRING, ""); + properties.setProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, ""); + properties.setProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, ""); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .initializeWithProperties(properties) .build(); @@ -298,7 +300,7 @@ public void testInitializeWithPropertiesNullValues() { Properties properties = new Properties(); // Properties with null values (getProperty returns default empty string) - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .initializeWithProperties(properties) .build(); @@ -309,7 +311,7 @@ public void testInitializeWithPropertiesNullValues() { @Test public void testGetBlobContainerServicePrincipalPath() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -329,7 +331,7 @@ public void testGetBlobContainerServicePrincipalPath() { @Test public void testGetBlobContainerSasTokenPath() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withSasToken("sv=2020-08-04&ss=b&srt=sco&sp=rwdlacx&se=2023-12-31T23:59:59Z&st=2023-01-01T00:00:00Z&spr=https&sig=test") @@ -348,7 +350,7 @@ public void testGetBlobContainerSasTokenPath() { @Test public void testGenerateSharedAccessSignatureServicePrincipalPath() throws DataStoreException, URISyntaxException, InvalidKeyException{ - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -377,7 +379,7 @@ public void testGenerateSharedAccessSignatureServicePrincipalPath() throws DataS @Test public void testGenerateSharedAccessSignatureAccountKeyPath() throws DataStoreException, URISyntaxException, InvalidKeyException { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withAccountKey("testkey") @@ -396,10 +398,10 @@ public void testGenerateSharedAccessSignatureAccountKeyPath() throws DataStoreEx @Test public void testBuilderStaticMethod() { - AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder("test-container"); + AzureBlobContainerProviderV12.Builder builder = AzureBlobContainerProviderV12.Builder.builder("test-container"); assertNotNull("Builder should not be null", builder); - AzureBlobContainerProvider provider = builder.build(); + AzureBlobContainerProviderV12 provider = builder.build(); assertNotNull("Provider should not be null", provider); assertEquals("Container name should match", "test-container", provider.getContainerName()); } @@ -407,19 +409,19 @@ public void testBuilderStaticMethod() { @Test public void testBuilderConstructorAccess() throws Exception { // Test that Builder constructor is private by accessing it via reflection - java.lang.reflect.Constructor constructor = - AzureBlobContainerProvider.Builder.class.getDeclaredConstructor(String.class); + java.lang.reflect.Constructor constructor = + AzureBlobContainerProviderV12.Builder.class.getDeclaredConstructor(String.class); assertFalse("Constructor should not be public", constructor.isAccessible()); // Make it accessible and test constructor.setAccessible(true); - AzureBlobContainerProvider.Builder builder = constructor.newInstance("test-container"); + AzureBlobContainerProviderV12.Builder builder = constructor.newInstance("test-container"); assertNotNull("Builder should not be null", builder); } @Test public void testDefaultEndpointSuffixUsage() throws Exception { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -428,7 +430,7 @@ public void testDefaultEndpointSuffixUsage() throws Exception { .build(); // Use reflection to access the DEFAULT_ENDPOINT_SUFFIX constant - Field defaultEndpointField = AzureBlobContainerProvider.class.getDeclaredField("DEFAULT_ENDPOINT_SUFFIX"); + Field defaultEndpointField = AzureBlobContainerProviderV12.class.getDeclaredField("DEFAULT_ENDPOINT_SUFFIX"); defaultEndpointField.setAccessible(true); String defaultEndpoint = (String) defaultEndpointField.get(null); assertEquals("Default endpoint should be core.windows.net", "core.windows.net", defaultEndpoint); @@ -448,7 +450,7 @@ public void testDefaultEndpointSuffixUsage() throws Exception { @Test public void testGenerateUserDelegationKeySignedSasWithMockTime() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -476,7 +478,7 @@ public void testGenerateUserDelegationKeySignedSasWithMockTime() { public void testGetBlobContainerWithNullRetryOptions() throws DataStoreException { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -491,7 +493,7 @@ public void testGetBlobContainerWithNullRetryOptions() throws DataStoreException public void testGetBlobContainerWithEmptyProperties() throws DataStoreException { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -505,7 +507,7 @@ public void testGetBlobContainerWithEmptyProperties() throws DataStoreException @Test public void testBuilderWithNullValues() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(null) .withAccountName(null) @@ -526,7 +528,7 @@ public void testBuilderWithNullValues() { public void testInitializeWithPropertiesNullProperties() { // Test with null properties object try { - AzureBlobContainerProvider.Builder.builder(CONTAINER_NAME) + AzureBlobContainerProviderV12.Builder.builder(CONTAINER_NAME) .initializeWithProperties(null); fail("Should throw NullPointerException with null properties"); } catch (NullPointerException e) { @@ -538,7 +540,7 @@ public void testInitializeWithPropertiesNullProperties() { @Test public void testAuthenticationPriorityOrder() throws Exception { // Test that connection string takes priority over other authentication methods - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(getConnectionString()) .withAccountName("testaccount") @@ -557,7 +559,7 @@ public void testAuthenticationPriorityOrder() throws Exception { @Test public void testGetBlobContainerWithAccountKeyFallback() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withAccountKey("testkey") @@ -575,7 +577,7 @@ public void testGetBlobContainerWithAccountKeyFallback() { @Test public void testAuthenticateViaServicePrincipalTrue() throws Exception { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -590,7 +592,7 @@ public void testAuthenticateViaServicePrincipalTrue() throws Exception { @Test public void testAuthenticateViaServicePrincipalFalseWithConnectionString() throws Exception { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString("connection-string") .withAccountName("testaccount") @@ -606,7 +608,7 @@ public void testAuthenticateViaServicePrincipalFalseWithConnectionString() throw @Test public void testAuthenticateViaServicePrincipalFalseWithMissingCredentials() throws Exception { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -621,7 +623,7 @@ public void testAuthenticateViaServicePrincipalFalseWithMissingCredentials() thr @Test public void testGetClientSecretCredential() throws Exception { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withTenantId("tenant-id") .withClientId("client-id") @@ -635,7 +637,7 @@ public void testGetClientSecretCredential() throws Exception { @Test public void testGetBlobContainerFromServicePrincipals() throws Exception { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -660,7 +662,7 @@ public void testGetBlobContainerFromServicePrincipals() throws Exception { @Test public void testGenerateUserDelegationKeySignedSas() { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withTenantId("tenant-id") @@ -688,7 +690,7 @@ public void testGenerateUserDelegationKeySignedSas() { @Test public void testGenerateSas() throws Exception { - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAccountName("testaccount") .withAccountKey("testkey") @@ -707,7 +709,7 @@ public void testGenerateSas() throws Exception { @Test public void testBuilderFieldAccess() throws Exception { - AzureBlobContainerProvider.Builder builder = AzureBlobContainerProvider.Builder.builder(CONTAINER_NAME); + AzureBlobContainerProviderV12.Builder builder = AzureBlobContainerProviderV12.Builder.builder(CONTAINER_NAME); // Test all builder methods and verify fields are set correctly builder.withAzureConnectionString("conn-string") @@ -719,38 +721,38 @@ public void testBuilderFieldAccess() throws Exception { .withClientId("client") .withClientSecret("secret"); - AzureBlobContainerProvider provider = builder.build(); + AzureBlobContainerProviderV12 provider = builder.build(); // Use reflection to verify all fields are set assertEquals("Container name should match", CONTAINER_NAME, provider.getContainerName()); assertEquals("Connection string should match", "conn-string", provider.getAzureConnectionString()); // Test private fields using reflection - Field accountNameField = AzureBlobContainerProvider.class.getDeclaredField("accountName"); + Field accountNameField = AzureBlobContainerProviderV12.class.getDeclaredField("accountName"); accountNameField.setAccessible(true); assertEquals("Account name should match", "account", accountNameField.get(provider)); - Field blobEndpointField = AzureBlobContainerProvider.class.getDeclaredField("blobEndpoint"); + Field blobEndpointField = AzureBlobContainerProviderV12.class.getDeclaredField("blobEndpoint"); blobEndpointField.setAccessible(true); assertEquals("Blob endpoint should match", "endpoint", blobEndpointField.get(provider)); - Field sasTokenField = AzureBlobContainerProvider.class.getDeclaredField("sasToken"); + Field sasTokenField = AzureBlobContainerProviderV12.class.getDeclaredField("sasToken"); sasTokenField.setAccessible(true); assertEquals("SAS token should match", "sas", sasTokenField.get(provider)); - Field accountKeyField = AzureBlobContainerProvider.class.getDeclaredField("accountKey"); + Field accountKeyField = AzureBlobContainerProviderV12.class.getDeclaredField("accountKey"); accountKeyField.setAccessible(true); assertEquals("Account key should match", "key", accountKeyField.get(provider)); - Field tenantIdField = AzureBlobContainerProvider.class.getDeclaredField("tenantId"); + Field tenantIdField = AzureBlobContainerProviderV12.class.getDeclaredField("tenantId"); tenantIdField.setAccessible(true); assertEquals("Tenant ID should match", "tenant", tenantIdField.get(provider)); - Field clientIdField = AzureBlobContainerProvider.class.getDeclaredField("clientId"); + Field clientIdField = AzureBlobContainerProviderV12.class.getDeclaredField("clientId"); clientIdField.setAccessible(true); assertEquals("Client ID should match", "client", clientIdField.get(provider)); - Field clientSecretField = AzureBlobContainerProvider.class.getDeclaredField("clientSecret"); + Field clientSecretField = AzureBlobContainerProviderV12.class.getDeclaredField("clientSecret"); clientSecretField.setAccessible(true); assertEquals("Client secret should match", "secret", clientSecretField.get(provider)); } @@ -768,7 +770,7 @@ private String getConnectionString() { @Test public void testGenerateSharedAccessSignatureWithoutHeaders() throws Exception { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -795,7 +797,7 @@ public void testGenerateSharedAccessSignatureWithoutHeaders() throws Exception { @Test public void testGenerateSharedAccessSignatureWithHeaders() throws Exception { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -808,7 +810,7 @@ public void testGenerateSharedAccessSignatureWithHeaders() throws Exception { // Generate SAS with headers BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true); - BlobSasHeaders headers = new BlobSasHeaders() + BlobSasHeadersV12 headers = new BlobSasHeadersV12() .setCacheControl("private, max-age=3600, immutable") .setContentType("image/png") .setContentDisposition("attachment; filename=\"test.png\""); @@ -836,7 +838,7 @@ public void testGenerateSharedAccessSignatureWithHeaders() throws Exception { @Test public void testGenerateSharedAccessSignatureWithNullHeaders() throws Exception { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -863,7 +865,7 @@ public void testGenerateSharedAccessSignatureWithNullHeaders() throws Exception @Test public void testGenerateSharedAccessSignatureWithPartialHeaders() throws Exception { String connectionString = getConnectionString(); - provider = AzureBlobContainerProvider.Builder + provider = AzureBlobContainerProviderV12.Builder .builder(CONTAINER_NAME) .withAzureConnectionString(connectionString) .build(); @@ -876,7 +878,7 @@ public void testGenerateSharedAccessSignatureWithPartialHeaders() throws Excepti // Generate SAS with only content-type header BlobSasPermission permissions = new BlobSasPermission().setReadPermission(true); - BlobSasHeaders headers = new BlobSasHeaders().setContentType("application/json"); + BlobSasHeadersV12 headers = new BlobSasHeadersV12().setContentType("application/json"); String sas = provider.generateSharedAccessSignature( null, blobName, permissions, 3600, new Properties(), headers); diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobStoreBackendV12IT.java similarity index 85% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobStoreBackendV12IT.java index 610fbc84bc9..76868dd95b2 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureBlobStoreBackendV12IT.java @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -75,24 +77,24 @@ import java.util.stream.StreamSupport; import static java.util.stream.Collectors.toSet; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BlOB_META_DIR_NAME; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_CONNECTION_STRING; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_CONTAINER_NAME; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_STORAGE_ACCOUNT_NAME; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_ENDPOINT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_CREATE_CONTAINER; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_REF_ON_INIT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_ENDPOINT; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_META_DIR_NAME; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_BLOB_REF_KEY; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_CONNECTION_STRING; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_CREATE_CONTAINER; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_REF_ON_INIT; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; /** - * Comprehensive test class for AzureBlobStoreBackend covering all methods and functionality. + * Comprehensive test class for AzureBlobStoreBackendV12 covering all methods and functionality. * Combines unit tests and integration tests. */ -public class AzureBlobStoreBackendTest { +public class AzureBlobStoreBackendV12IT { @ClassRule public static AzuriteDockerRule azurite = new AzuriteDockerRule(); @@ -102,7 +104,7 @@ public class AzureBlobStoreBackendTest { private static final Set BLOBS = Set.of("blob1", "blob2"); private BlobContainerClient container; - private AzureBlobStoreBackend backend; + private AzureBlobStoreBackendV12 backend; private Properties testProperties; @Before @@ -116,7 +118,7 @@ public void setUp() { testProperties = createTestProperties(); // Create backend instance - backend = new AzureBlobStoreBackend(); + backend = new AzureBlobStoreBackendV12(); backend.setProperties(testProperties); } @@ -150,7 +152,7 @@ private Properties createTestProperties() { } private static String getConnectionString() { - return Utils.getConnectionString( + return UtilsV12.getConnectionString( AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint() @@ -172,7 +174,7 @@ public void testInitWithValidProperties() throws Exception { @Test public void testInitWithNullProperties() { - AzureBlobStoreBackend nullPropsBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 nullPropsBackend = new AzureBlobStoreBackendV12(); // Should not set properties, will try to read from default config file try { @@ -194,7 +196,7 @@ public void testInitWithNullPropertiesAndValidConfigFile() throws Exception { configProps.store(fos, "Test configuration for null properties test"); } - AzureBlobStoreBackend nullPropsBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 nullPropsBackend = new AzureBlobStoreBackendV12(); // Don't set properties - should read from azure.properties file try { @@ -228,7 +230,7 @@ public void testSetProperties() { // Verify properties were set (using reflection to access private field) try { - Field propertiesField = AzureBlobStoreBackend.class.getDeclaredField("properties"); + Field propertiesField = AzureBlobStoreBackendV12.class.getDeclaredField("properties"); propertiesField.setAccessible(true); Properties actualProps = (Properties) propertiesField.get(backend); assertEquals("Properties should be set", "test.value", actualProps.getProperty("test.key")); @@ -243,7 +245,7 @@ public void testConcurrentRequestCountValidation() throws Exception { Properties lowProps = createTestProperties(); lowProps.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1"); - AzureBlobStoreBackend lowBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 lowBackend = new AzureBlobStoreBackendV12(); lowBackend.setProperties(lowProps); lowBackend.init(); @@ -255,7 +257,7 @@ public void testConcurrentRequestCountValidation() throws Exception { Properties highProps = createTestProperties(); highProps.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "100"); - AzureBlobStoreBackend highBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 highBackend = new AzureBlobStoreBackendV12(); highBackend.setProperties(highProps); highBackend.init(); @@ -296,11 +298,46 @@ public void testGetAzureContainerThreadSafety() throws Exception { executor.shutdown(); } + @Test + public void testGetAzureContainerInitializesProviderOnlyOnce() throws Exception { + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); + testBackend.setProperties(testProperties); + + Field providerField = AzureBlobStoreBackendV12.class.getDeclaredField("azureBlobContainerProvider"); + providerField.setAccessible(true); + + BlobContainerClient containerClient = container; + AzureBlobContainerProviderV12 mockProvider = org.mockito.Mockito.mock(AzureBlobContainerProviderV12.class); + org.mockito.Mockito.when(mockProvider.getBlobContainer(any(), any())).thenReturn(containerClient); + providerField.set(testBackend, mockProvider); + + int threadCount = 8; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + List> futures = new ArrayList<>(); + try { + for (int i = 0; i < threadCount; i++) { + futures.add(executor.submit(() -> { + latch.countDown(); + latch.await(); + return testBackend.getAzureContainer(); + })); + } + for (Future future : futures) { + assertSame(containerClient, future.get(5, TimeUnit.SECONDS)); + } + org.mockito.Mockito.verify(mockProvider, org.mockito.Mockito.times(1)).getBlobContainer(any(), any()); + } finally { + executor.shutdownNow(); + testBackend.close(); + } + } + @Test public void testGetAzureContainerWhenNull() throws Exception { // Create a backend with valid properties but don't initialize it // This ensures azureContainer field remains null initially - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(testProperties); // Initialize the backend to set up azureBlobContainerProvider @@ -308,7 +345,7 @@ public void testGetAzureContainerWhenNull() throws Exception { try { // Reset azureContainer to null using reflection to test the null case - Field azureContainerReferenceField = AzureBlobStoreBackend.class.getDeclaredField("azureContainerReference"); + Field azureContainerReferenceField = AzureBlobStoreBackendV12.class.getDeclaredField("azureContainerReference"); azureContainerReferenceField.setAccessible(true); @SuppressWarnings("unchecked") AtomicReference azureContainerReference = (AtomicReference) azureContainerReferenceField.get(testBackend); @@ -342,15 +379,15 @@ public void testGetAzureContainerWhenNull() throws Exception { @Test public void testGetAzureContainerWithProviderException() throws Exception { // Create a backend with a mock provider that throws exception - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(testProperties); // Set up mock provider using reflection - Field providerField = AzureBlobStoreBackend.class.getDeclaredField("azureBlobContainerProvider"); + Field providerField = AzureBlobStoreBackendV12.class.getDeclaredField("azureBlobContainerProvider"); providerField.setAccessible(true); // Create mock provider that throws DataStoreException - AzureBlobContainerProvider mockProvider = org.mockito.Mockito.mock(AzureBlobContainerProvider.class); + AzureBlobContainerProviderV12 mockProvider = org.mockito.Mockito.mock(AzureBlobContainerProviderV12.class); org.mockito.Mockito.when(mockProvider.getBlobContainer(any(), any())) .thenThrow(new DataStoreException("Mock connection failure")); @@ -364,7 +401,7 @@ public void testGetAzureContainerWithProviderException() throws Exception { assertEquals("Exception message should match", "Mock connection failure", e.getMessage()); // Verify azureContainer field remains null after exception - Field azureContainerField = AzureBlobStoreBackend.class.getDeclaredField("azureContainerReference"); + Field azureContainerField = AzureBlobStoreBackendV12.class.getDeclaredField("azureContainerReference"); azureContainerField.setAccessible(true); @SuppressWarnings("unchecked") BlobContainerClient containerAfterException = ((AtomicReference) azureContainerField.get(testBackend)).get(); @@ -808,14 +845,14 @@ public void testMetadataPrefixComposition() throws Exception { try { // Verify the blob is stored with the META/ prefix in Azure storage - String expectedBlobName = AzureConstants.AZURE_BLOB_META_KEY_PREFIX + recordName; + String expectedBlobName = AzureConstantsV12.AZURE_BLOB_META_KEY_PREFIX + recordName; BlobClient blobClient = azureContainer.getBlobClient(expectedBlobName); assertTrue("Blob should exist at path with META/ prefix: " + expectedBlobName, blobClient.exists()); // Verify the blob is listed under the META directory ListBlobsOptions listOptions = new ListBlobsOptions(); - listOptions.setPrefix(AzureConstants.AZURE_BlOB_META_DIR_NAME); + listOptions.setPrefix(AzureConstantsV12.AZURE_BLOB_META_DIR_NAME); boolean foundBlobWithMetaPrefix = false; for (BlobItem blobItem : azureContainer.listBlobs(listOptions, null)) { @@ -1013,7 +1050,7 @@ public void testMetadataRecordExists() throws Exception { public void testGetKeyName() throws Exception { // Test the static getKeyName method using reflection DataIdentifier identifier = new DataIdentifier("abcd1234567890"); - Method getKeyNameMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "getKeyName", DataIdentifier.class); + Method getKeyNameMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackendV12.class, "getKeyName", DataIdentifier.class); getKeyNameMethod.setAccessible(true); String keyName = (String) getKeyNameMethod.invoke(null, identifier); @@ -1023,7 +1060,7 @@ public void testGetKeyName() throws Exception { @Test public void testGetIdentifierName() throws Exception { // Test the static getIdentifierName method using reflection - Method getIdentifierNameMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "getIdentifierName", String.class); + Method getIdentifierNameMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackendV12.class, "getIdentifierName", String.class); getIdentifierNameMethod.setAccessible(true); String identifierName = (String) getIdentifierNameMethod.invoke(null, "abcd-1234567890"); @@ -1043,7 +1080,7 @@ public void testGetIdentifierName() throws Exception { @Test public void testAddMetaKeyPrefix() throws Exception { // Test the static addMetaKeyPrefix method using reflection - Method addMetaKeyPrefixMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "addMetaKeyPrefix", String.class); + Method addMetaKeyPrefixMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackendV12.class, "addMetaKeyPrefix", String.class); addMetaKeyPrefixMethod.setAccessible(true); String result = (String) addMetaKeyPrefixMethod.invoke(null, "test-key"); assertTrue("Result should contain META prefix", result.startsWith("META/")); @@ -1053,7 +1090,7 @@ public void testAddMetaKeyPrefix() throws Exception { @Test public void testStripMetaKeyPrefix() throws Exception { // Test the static stripMetaKeyPrefix method using reflection - Method stripMetaKeyPrefixMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackend.class, "stripMetaKeyPrefix", String.class); + Method stripMetaKeyPrefixMethod = MethodUtils.getMatchingMethod(AzureBlobStoreBackendV12.class, "stripMetaKeyPrefix", String.class); stripMetaKeyPrefixMethod.setAccessible(true); String withPrefix = "META/test-key"; @@ -1071,7 +1108,7 @@ public void testGetOrCreateReferenceKey() throws Exception { Properties propsWithRef = createTestProperties(); propsWithRef.setProperty(AZURE_REF_ON_INIT, "true"); - AzureBlobStoreBackend refBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 refBackend = new AzureBlobStoreBackendV12(); refBackend.setProperties(propsWithRef); refBackend.init(); @@ -1126,7 +1163,7 @@ public void testSetHttpDownloadURIExpirySeconds() throws Exception { MethodUtils.invokeMethod(backend, true, "setHttpDownloadURIExpirySeconds", 3600); // Verify the field was set - Field expiryField = AzureBlobStoreBackend.class.getDeclaredField("httpDownloadURIExpirySeconds"); + Field expiryField = AzureBlobStoreBackendV12.class.getDeclaredField("httpDownloadURIExpirySeconds"); expiryField.setAccessible(true); int expiry = (int) expiryField.get(backend); assertEquals("Expiry should be set", 3600, expiry); @@ -1138,7 +1175,7 @@ public void testSetHttpUploadURIExpirySeconds() throws Exception { MethodUtils.invokeMethod(backend, true, "setHttpUploadURIExpirySeconds", 1800); // Verify the field was set - Field expiryField = AzureBlobStoreBackend.class.getDeclaredField("httpUploadURIExpirySeconds"); + Field expiryField = AzureBlobStoreBackendV12.class.getDeclaredField("httpUploadURIExpirySeconds"); expiryField.setAccessible(true); int expiry = (int) expiryField.get(backend); assertEquals("Expiry should be set", 1800, expiry); @@ -1150,7 +1187,7 @@ public void testSetHttpDownloadURICacheSize() throws Exception { // Test with positive cache size MethodUtils.invokeMethod(backend, true, "setHttpDownloadURICacheSize", 100); - Field cacheField = AzureBlobStoreBackend.class.getDeclaredField("httpDownloadURICache"); + Field cacheField = AzureBlobStoreBackendV12.class.getDeclaredField("httpDownloadURICache"); cacheField.setAccessible(true); Cache cache = (Cache) cacheField.get(backend); assertNotNull("Cache should be created for positive size", cache); @@ -1169,7 +1206,7 @@ public void testCreateHttpDownloadURI() throws Exception { Properties propsWithDownload = createTestProperties(); propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 downloadBackend = new AzureBlobStoreBackendV12(); downloadBackend.setProperties(propsWithDownload); downloadBackend.init(); @@ -1342,7 +1379,7 @@ public void testClose() throws Exception { @Test public void testInitWithInvalidConnectionString() { - AzureBlobStoreBackend invalidBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 invalidBackend = new AzureBlobStoreBackendV12(); Properties invalidProps = new Properties(); invalidProps.setProperty(AZURE_CONNECTION_STRING, "invalid-connection-string"); invalidProps.setProperty(AZURE_BLOB_CONTAINER_NAME, "test-container"); @@ -1364,7 +1401,7 @@ public void testInitWithMissingContainer() { Properties propsNoContainer = createTestProperties(); propsNoContainer.remove(AZURE_BLOB_CONTAINER_NAME); - AzureBlobStoreBackend noContainerBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 noContainerBackend = new AzureBlobStoreBackendV12(); noContainerBackend.setProperties(propsNoContainer); try { @@ -1386,7 +1423,7 @@ public void testInitWithCreateContainerDisabled() throws Exception { propsNoCreate.setProperty(AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME + "-nocreate"); propsNoCreate.setProperty(AZURE_CREATE_CONTAINER, "false"); - AzureBlobStoreBackend noCreateBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 noCreateBackend = new AzureBlobStoreBackendV12(); noCreateBackend.setProperties(propsNoCreate); noCreateBackend.init(); @@ -1437,21 +1474,13 @@ public void testLargeFileHandling() throws Exception { public void testEmptyFileHandling() throws Exception { backend.init(); - // Create empty file File emptyFile = File.createTempFile("empty-test", ".tmp"); DataIdentifier identifier = new DataIdentifier("emptyfiletest123"); try { - // Azure SDK doesn't support zero-length block sizes, so this should throw an exception backend.write(identifier, emptyFile); - fail("Expected IllegalArgumentException for empty file"); - } catch (IllegalArgumentException e) { - // Expected - Azure SDK doesn't allow zero-length block sizes - assertTrue("Should mention block size", e.getMessage().contains("blockSize")); - } catch (Exception e) { - // Also acceptable if wrapped in another exception - assertTrue("Should be related to empty file handling", - e.getMessage().contains("blockSize") || e.getCause() instanceof IllegalArgumentException); + assertTrue("Empty file should be written successfully", backend.exists(identifier)); + backend.deleteRecord(identifier); } finally { emptyFile.delete(); } @@ -1560,14 +1589,14 @@ public void testMetadataDirectoryStructure() throws Exception { try { // Verify the record is stored with correct path prefix BlobContainerClient azureContainer = backend.getAzureContainer(); - String expectedBlobName = AZURE_BlOB_META_DIR_NAME + "/" + metadataName; + String expectedBlobName = AZURE_BLOB_META_DIR_NAME + "/" + metadataName; BlobClient blobClient = azureContainer.getBlobClient(expectedBlobName); assertTrue("Blob should exist at expected path", blobClient.exists()); // Verify the blob is in the META directory ListBlobsOptions listOptions = new ListBlobsOptions(); - listOptions.setPrefix(AZURE_BlOB_META_DIR_NAME); + listOptions.setPrefix(AZURE_BLOB_META_DIR_NAME); boolean foundBlob = false; for (BlobItem blobItem : azureContainer.listBlobs(listOptions, null)) { @@ -1642,7 +1671,7 @@ public void testConcurrentRequestCountTooLow() throws Exception { Properties props = createTestProperties(); props.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1"); // Below minimum - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1656,7 +1685,7 @@ public void testConcurrentRequestCountTooHigh() throws Exception { Properties props = createTestProperties(); props.setProperty(AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1000"); // Above maximum - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1667,9 +1696,9 @@ public void testConcurrentRequestCountTooHigh() throws Exception { @Test public void testRequestTimeoutConfiguration() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT, "30000"); + props.setProperty(AzureConstantsV12.AZURE_BLOB_REQUEST_TIMEOUT, "30000"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1679,9 +1708,9 @@ public void testRequestTimeoutConfiguration() throws Exception { @Test public void testPresignedDownloadURIVerifyExistsDisabled() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1696,7 +1725,7 @@ public void testCreateContainerDisabled() throws Exception { Properties props = createTestProperties(); props.setProperty(AZURE_CREATE_CONTAINER, "false"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1708,7 +1737,7 @@ public void testReferenceKeyInitializationDisabled() throws Exception { Properties props = createTestProperties(); props.setProperty(AZURE_REF_ON_INIT, "false"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1718,10 +1747,10 @@ public void testReferenceKeyInitializationDisabled() throws Exception { @Test public void testHttpDownloadURICacheConfiguration() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "100"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "100"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1731,10 +1760,10 @@ public void testHttpDownloadURICacheConfiguration() throws Exception { @Test public void testHttpDownloadURICacheDisabled() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); // No cache max size property - should default to 0 (disabled) - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1744,9 +1773,9 @@ public void testHttpDownloadURICacheDisabled() throws Exception { @Test public void testUploadDomainOverride() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, "custom-upload.example.com"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, "custom-upload.example.com"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1756,9 +1785,9 @@ public void testUploadDomainOverride() throws Exception { @Test public void testDownloadDomainOverride() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, "custom-download.example.com"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, "custom-download.example.com"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1824,9 +1853,9 @@ public void testInitiateHttpUploadInvalidParameters() throws Exception { @Test public void testInitiateHttpUploadSinglePutTooLarge() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1846,9 +1875,9 @@ public void testInitiateHttpUploadSinglePutTooLarge() throws Exception { @Test public void testInitiateHttpUploadWithValidParameters() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1869,9 +1898,9 @@ public void testInitiateHttpUploadWithValidParameters() throws Exception { @Test public void testInitiateHttpUploadPartSizeTooLarge() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1897,9 +1926,9 @@ public void testInitiateHttpUploadPartSizeTooLarge() throws Exception { @Test public void testCreateHttpDownloadURINullIdentifier() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1916,9 +1945,9 @@ public void testCreateHttpDownloadURINullIdentifier() throws Exception { @Test public void testCreateHttpDownloadURINullOptions() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1935,10 +1964,10 @@ public void testCreateHttpDownloadURINullOptions() throws Exception { @Test public void testCreateHttpDownloadURIForNonExistentBlob() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "true"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "true"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -1991,7 +2020,7 @@ public void testBlobStorageExceptionHandling() { invalidProps.setProperty(AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); invalidProps.setProperty(AZURE_CONNECTION_STRING, "invalid-connection-string"); - AzureBlobStoreBackend invalidBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 invalidBackend = new AzureBlobStoreBackendV12(); invalidBackend.setProperties(invalidProps); try { @@ -2009,11 +2038,11 @@ public void testBlobStorageExceptionHandling() { @Test public void testHttpDownloadURICacheHit() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "10"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "10"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -2051,7 +2080,7 @@ public void testHttpDownloadURIWithoutExpiry() throws Exception { Properties props = createTestProperties(); // Don't set expiry seconds - should default to 0 (disabled) - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -2065,9 +2094,9 @@ public void testHttpDownloadURIWithoutExpiry() throws Exception { @Test public void testCompleteHttpUploadWithMissingRecord() throws Exception { Properties props = createTestProperties(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend testBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 testBackend = new AzureBlobStoreBackendV12(); testBackend.setProperties(props); testBackend.init(); @@ -2097,7 +2126,7 @@ public void initWithSharedAccessSignature_readOnly() throws Exception { BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions); String sasToken = container.generateSas(sasValues); - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken)); azureBlobStoreBackend.init(); @@ -2119,7 +2148,7 @@ public void initWithSharedAccessSignature_readWrite() throws Exception { BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions); String sasToken = container.generateSas(sasValues); - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken)); azureBlobStoreBackend.init(); @@ -2140,7 +2169,7 @@ public void connectWithSharedAccessSignatureURL_expired() throws Exception { BlobServiceSasSignatureValues sasValues = new BlobServiceSasSignatureValues(expiryTime, permissions); String sasToken = container.generateSas(sasValues); - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken)); azureBlobStoreBackend.init(); @@ -2151,7 +2180,7 @@ public void connectWithSharedAccessSignatureURL_expired() throws Exception { @Test public void initWithAccessKey() throws Exception { - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithAccessKey()); azureBlobStoreBackend.init(); @@ -2162,7 +2191,7 @@ public void initWithAccessKey() throws Exception { @Test public void initWithConnectionURL() throws Exception { - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString()); azureBlobStoreBackend.init(); @@ -2173,7 +2202,7 @@ public void initWithConnectionURL() throws Exception { @Test public void initSecret() throws Exception { - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString()); azureBlobStoreBackend.init(); @@ -2185,7 +2214,7 @@ public void initWithServicePrincipals() throws Exception { // Create blob container with test blobs using Azurite createBlobContainer(); - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString()); azureBlobStoreBackend.init(); @@ -2198,7 +2227,7 @@ public void initWithServicePrincipals() throws Exception { public void testMetadataOperationsWithRenamedConstants() throws Exception { createBlobContainer(); - AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 azureBlobStoreBackend = new AzureBlobStoreBackendV12(); azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString()); azureBlobStoreBackend.init(); @@ -2251,40 +2280,40 @@ private BlobContainerClient createBlobContainer() { private static Properties getConfigurationWithSasToken(String sasToken) { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_SAS, sasToken); - properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "false"); - properties.setProperty(AzureConstants.AZURE_REF_ON_INIT, "false"); + properties.setProperty(AzureConstantsV12.AZURE_SAS, sasToken); + properties.setProperty(AzureConstantsV12.AZURE_CREATE_CONTAINER, "false"); + properties.setProperty(AzureConstantsV12.AZURE_REF_ON_INIT, "false"); return properties; } private static Properties getConfigurationWithAccessKey() { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, AzuriteDockerRule.ACCOUNT_KEY); + properties.setProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, AzuriteDockerRule.ACCOUNT_KEY); return properties; } @NotNull private static Properties getConfigurationWithConnectionString() { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); + properties.setProperty(AzureConstantsV12.AZURE_CONNECTION_STRING, getConnectionString()); return properties; } @NotNull private static Properties getBasicConfiguration() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); - properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); - properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, ""); + properties.setProperty(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); + properties.setProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); + properties.setProperty(AzureConstantsV12.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); + properties.setProperty(AzureConstantsV12.AZURE_CREATE_CONTAINER, ""); return properties; } - private static void assertReadAccessGranted(AzureBlobStoreBackend backend, Set expectedBlobs) throws Exception { + private static void assertReadAccessGranted(AzureBlobStoreBackendV12 backend, Set expectedBlobs) throws Exception { BlobContainerClient container = backend.getAzureContainer(); Set actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false) .map(blobItem -> container.getBlobClient(blobItem.getName()).getBlobName()) - .filter(name -> !name.contains(AZURE_BlOB_META_DIR_NAME)) + .filter(name -> !name.contains(AZURE_BLOB_META_DIR_NAME)) .collect(toSet()); Set expectedBlobNames = expectedBlobs.stream().map(name -> name + ".txt").collect(toSet()); @@ -2303,14 +2332,14 @@ private static void assertReadAccessGranted(AzureBlobStoreBackend backend, Set concat(Set set, String element) { return Stream.concat(set.stream(), Stream.of(element)).collect(Collectors.toSet()); } - private static void assertReferenceSecret(AzureBlobStoreBackend AzureBlobStoreBackend) + private static void assertReferenceSecret(AzureBlobStoreBackendV12 AzureBlobStoreBackendV12) throws DataStoreException { // assert secret already created on init - DataRecord refRec = AzureBlobStoreBackend.getMetadataRecord("reference.key"); + DataRecord refRec = AzureBlobStoreBackendV12.getMetadataRecord("reference.key"); assertNotNull("Reference data record null", refRec); assertTrue("reference key is empty", refRec.getLength() > 0); } @@ -2350,7 +2379,7 @@ public void testCreateHttpDownloadURIWithHeaders() throws Exception { Properties propsWithDownload = createTestProperties(); propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 downloadBackend = new AzureBlobStoreBackendV12(); downloadBackend.setProperties(propsWithDownload); downloadBackend.init(); @@ -2411,7 +2440,7 @@ public void testCreateHttpDownloadURIWithDefaultHeaders() throws Exception { Properties propsWithDownload = createTestProperties(); propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 downloadBackend = new AzureBlobStoreBackendV12(); downloadBackend.setProperties(propsWithDownload); downloadBackend.init(); @@ -2448,7 +2477,7 @@ public void testCreateHttpDownloadURIWithContentDisposition() throws Exception { Properties propsWithDownload = createTestProperties(); propsWithDownload.setProperty(PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - AzureBlobStoreBackend downloadBackend = new AzureBlobStoreBackend(); + AzureBlobStoreBackendV12 downloadBackend = new AzureBlobStoreBackendV12(); downloadBackend.setProperties(propsWithDownload); downloadBackend.init(); @@ -2483,4 +2512,127 @@ public void testCreateHttpDownloadURIWithContentDisposition() throws Exception { downloadBackend.close(); } } + + // ===================================================================== + // CSO Prevention Tests (Step 3.3) + // + // These tests guard against constant drift in V12 and ensure V12 + // values are not accidentally set to V8 values (or vice versa). + // ===================================================================== + + @Test + public void testV12MaxPartSize_MustBe4000MiB() { + assertEquals("V12 MAX_MULTIPART_UPLOAD_PART_SIZE must be 4000 MiB", + 4000L * 1024 * 1024, + AzureConstantsV12.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE); + } + + @Test + public void testV12MinPartSize_MustBe4MiB() { + assertEquals("V12 MIN_MULTIPART_UPLOAD_PART_SIZE must be 4 MiB", + 4L * 1024 * 1024, + AzureConstantsV12.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE); + } + + @Test + public void testV12MaxBinaryUploadSize_MustBe190TiB() { + assertEquals("V12 MAX_BINARY_UPLOAD_SIZE must be 190 TiB", + 190L * 1024 * 1024 * 1024 * 1024, + AzureConstantsV12.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE); + } + + @Test + public void testV12DefaultConcurrentRequestCount_MustBe5() { + assertEquals("V12 DEFAULT_CONCURRENT_REQUEST_COUNT must be 5", + 5, + AzureConstantsV12.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); + } + + @Test + public void testV12MaxConcurrentRequestCount_MustBe10() { + assertEquals("V12 MAX_CONCURRENT_REQUEST_COUNT must be 10", + 10, + AzureConstantsV12.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT); + } + + @Test + public void testV12InitiateHttpUpload_100MiB_CorrectURIs() throws Exception { + container = azurite.getContainer(CONTAINER_NAME, getConnectionString()); + + AzureBlobStoreBackendV12 backend = new AzureBlobStoreBackendV12(); + Properties props = getConfigurationWithConnectionString(); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60"); + backend.setProperties(props); + backend.init(); + + DataRecordUploadOptions uploadOptions = DataRecordUploadOptions.DEFAULT; + + // 100 MiB with unlimited URIs: ceil(100MiB / 4MiB minPart) = 25 URIs + DataRecordUpload upload = backend.initiateHttpUpload(100L * 1024 * 1024, -1, uploadOptions); + + assertNotNull("Upload should not be null", upload); + long expectedParts = (long) Math.ceil( + ((double) (100L * 1024 * 1024)) / ((double) AzureConstantsV12.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); + assertEquals("100 MiB upload with unlimited URIs should produce ceil(100MiB/4MiB) = " + expectedParts + " URIs", + expectedParts, upload.getUploadURIs().size()); + } + + @Test + public void testV12InitiateHttpUpload_1GiB_CorrectURIs() throws Exception { + container = azurite.getContainer(CONTAINER_NAME, getConnectionString()); + + AzureBlobStoreBackendV12 backend = new AzureBlobStoreBackendV12(); + Properties props = getConfigurationWithConnectionString(); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60"); + backend.setProperties(props); + backend.init(); + + DataRecordUploadOptions uploadOptions = DataRecordUploadOptions.DEFAULT; + + long oneGiB = 1024L * 1024 * 1024; + // ceil(1GiB / 4MiB) = 256 URIs + DataRecordUpload upload = backend.initiateHttpUpload(oneGiB, -1, uploadOptions); + + assertNotNull("Upload should not be null", upload); + long expectedParts = (long) Math.ceil( + ((double) oneGiB) / ((double) AzureConstantsV12.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); + int uriCount = upload.getUploadURIs().size(); + assertEquals("1 GiB upload should produce ceil(1GiB/4MiB) = " + expectedParts + " URIs", + expectedParts, uriCount); + // Must NOT be ~4000 URIs (which would happen with old 256KiB min) + assertTrue("1 GiB upload must produce reasonable URI count (got " + uriCount + "), not ~4000", + uriCount < 1000); + } + + @Test + public void testV12InitiateHttpUpload_40GiB_CorrectURIs() throws Exception { + // 40 GiB is a realistic DAM archive size that triggered the production OOM on V8. + // V12 with correct 4 MiB min part: ceil(40 GiB / 4 MiB) = 10240 URIs. + // With old 256 KiB min: would be ~163840 URIs — wastefully many. + container = azurite.getContainer(CONTAINER_NAME, getConnectionString()); + + AzureBlobStoreBackendV12 backend = new AzureBlobStoreBackendV12(); + Properties props = getConfigurationWithConnectionString(); + props.setProperty(AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60"); + backend.setProperties(props); + backend.init(); + + DataRecordUploadOptions uploadOptions = DataRecordUploadOptions.DEFAULT; + + long fortyGiB = 40L * 1024 * 1024 * 1024; + DataRecordUpload upload = backend.initiateHttpUpload(fortyGiB, -1, uploadOptions); + + assertNotNull("Upload should not be null", upload); + long expectedParts = (long) Math.ceil( + ((double) fortyGiB) / ((double) AzureConstantsV12.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); + int uriCount = upload.getUploadURIs().size(); + assertEquals("40 GiB upload should produce ceil(40GiB/4MiB) = " + expectedParts + " URIs", + expectedParts, uriCount); + assertTrue("40 GiB upload must produce thousands of URIs (got " + uriCount + ")", + uriCount > 1000); + // Must NOT be ~163840 URIs (which would happen with old 256KiB min) + assertTrue("40 GiB upload must not produce excessive URI count (got " + uriCount + ")", + uriCount < 50000); + } + } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderCDNTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderCDNV12Test.java similarity index 69% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderCDNTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderCDNV12Test.java index 3f5c54cc102..bbb4a462f57 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderCDNTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderCDNV12Test.java @@ -17,11 +17,13 @@ * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; + +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12.AzureConstantsV12.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; @@ -46,44 +48,37 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; -public class AzureDataRecordAccessProviderCDNTest extends AzureDataRecordAccessProviderTest { +public class AzureDataRecordAccessProviderCDNV12Test extends AzureDataRecordAccessProviderV12Test { @ClassRule public static TemporaryFolder homeDir = new TemporaryFolder(new File("target")); private static AzureDataStore cdnDataStore; - private static String DOWNLOAD_URI_DOMAIN = AzureDataStoreUtils - .getDirectAccessDataStoreProperties() - .getProperty(PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); - private static String UPLOAD_URI_DOMAIN = AzureDataStoreUtils - .getDirectAccessDataStoreProperties() - .getProperty(PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null); - - private static String cdnSetupNotice = String.format( - "%s\n%s %s '%s' %s '%s' %s %s", - "No override domains configured - skipping Azure CDN tests.", - "To run these tests, set up an Azure CDN in the Azure console or command line,", - "then set the CDN domain as the property value for", - PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, - "and/or", - PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, - "in your Azure configuration file, and then provide this file to the", - "test via the -Dazure.config command-line switch" - ); + private static final String cdnSetupNotice = + "No override domains configured - skipping Azure CDN tests. " + + "To run these tests, set up an Azure CDN in the Azure console or command line, " + + "then set the CDN domain as the property value for '" + + PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE + "' and/or '" + + PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE + + "' in your Azure configuration file, and then provide this file to the " + + "test via the -Dazure.config command-line switch"; @BeforeClass public static void setupDataStore() throws Exception { assumeTrue(cdnSetupNotice, isCDNConfigured()); - cdnDataStore = AzureDataStoreUtils.setupDirectAccessDataStore(homeDir, + cdnDataStore = setupDirectAccessDataStore(homeDir, expirySeconds, expirySeconds); } private static boolean isCDNConfigured() { - return ! StringUtils.isEmpty(DOWNLOAD_URI_DOMAIN) && ! StringUtils.isEmpty(UPLOAD_URI_DOMAIN); + Properties props = getDirectAccessDataStoreProperties(); + String downloadDomain = props.getProperty(PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); + String uploadDomain = props.getProperty(PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null); + return !StringUtils.isEmpty(downloadDomain) && !StringUtils.isEmpty(uploadDomain); } private static AzureDataStore createDataStore(@NotNull Properties properties) throws Exception { - return AzureDataStoreUtils.setupDirectAccessDataStore(homeDir, expirySeconds, expirySeconds, properties); + return setupDirectAccessDataStore(homeDir, expirySeconds, expirySeconds, properties); } @Override @@ -93,7 +88,7 @@ protected ConfigurableDataRecordAccessProvider getDataStore() { @Override protected ConfigurableDataRecordAccessProvider getDataStore(@NotNull Properties overrideProperties) throws Exception { - return createDataStore(AzureDataStoreUtils.getDirectAccessDataStoreProperties(overrideProperties)); + return createDataStore(getDirectAccessDataStoreProperties(overrideProperties)); } private ConfigurableDataRecordAccessProvider getCDNEnabledDataStore() throws Exception { @@ -109,23 +104,29 @@ public void testCDNDownloadURIContainsDownloadDomain() throws Exception { DataIdentifier id = new DataIdentifier("identifier"); URI downloadUri = ds.getDownloadURI(id, DataRecordDownloadOptions.DEFAULT); assertNotNull(downloadUri); - assertEquals(DOWNLOAD_URI_DOMAIN, downloadUri.getHost()); + String expectedDownloadDomain = getDirectAccessDataStoreProperties() + .getProperty(PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); + assertEquals(expectedDownloadDomain, downloadUri.getHost()); } @Test public void testCDNUploadURIContainsUploadDomain() throws Exception { + String expectedUploadDomain = getDirectAccessDataStoreProperties() + .getProperty(PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null); Properties properties = new Properties(); ConfigurableDataRecordAccessProvider ds = getDataStore(properties); DataRecordUpload upload = ds.initiateDataRecordUpload(ONE_MB, 10); assertNotNull(upload); assertTrue(upload.getUploadURIs().size() > 0); for (URI uri : upload.getUploadURIs()) { - assertEquals(UPLOAD_URI_DOMAIN, uri.getHost()); + assertEquals(expectedUploadDomain, uri.getHost()); } } @Test public void testVetoDownloadDomainOverride() throws Exception { + String downloadDomain = getDirectAccessDataStoreProperties() + .getProperty(PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null); ConfigurableDataRecordAccessProvider ds = getCDNEnabledDataStore(); DataIdentifier id = new DataIdentifier("identifier"); DataRecordDownloadOptions options = @@ -134,10 +135,10 @@ public void testVetoDownloadDomainOverride() throws Exception { ); URI downloadUri = ds.getDownloadURI(id, options); assertNotNull(downloadUri); - assertNotEquals(DOWNLOAD_URI_DOMAIN, downloadUri.getHost()); + assertNotEquals(downloadDomain, downloadUri.getHost()); - Properties properties = AzureDataStoreUtils.getDirectAccessDataStoreProperties(); - String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, null); + Properties properties = getDirectAccessDataStoreProperties(); + String accountName = properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, null); assertNotNull(accountName); assertEquals(String.format("%s.blob.core.windows.net", accountName), downloadUri.getHost()); } @@ -150,8 +151,8 @@ public void testVetoUploadDomainOverride() throws Exception { assertNotNull(upload); assertTrue(upload.getUploadURIs().size() > 0); - Properties properties = AzureDataStoreUtils.getDirectAccessDataStoreProperties(); - String accountName = properties.getProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, null); + Properties properties = getDirectAccessDataStoreProperties(); + String accountName = properties.getProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, null); assertNotNull(accountName); String defaultDomain = String.format("%s.blob.core.windows.net", accountName); for (URI uri : upload.getUploadURIs()) { diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderV12IT.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderV12IT.java new file mode 100644 index 00000000000..9d58d6dcc09 --- /dev/null +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderV12IT.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.Properties; + +import javax.net.ssl.HttpsURLConnection; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; +import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; +import org.apache.jackrabbit.oak.spi.blob.data.DataStore; +import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; +import org.apache.jackrabbit.oak.commons.PropertiesUtil; +import org.apache.jackrabbit.oak.commons.collections.MapUtils; +import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils; +import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.AbstractDataRecordAccessProviderIT; +import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assume.assumeTrue; + +/** + * As the test is memory intensive requires -Dtest.opts.memory=-Xmx2G + */ +public class AzureDataRecordAccessProviderV12IT extends AbstractDataRecordAccessProviderIT { + + private static final Logger log = LoggerFactory.getLogger(AzureDataRecordAccessProviderV12IT.class); + + private static final String DEFAULT_CONFIG_PATH = "./src/test/resources/azure.properties"; + private static final String DEFAULT_PROPERTY_FILE = "azure.properties"; + private static final String SYS_PROP_NAME = "azure.config"; + + @ClassRule + public static TemporaryFolder homeDir = new TemporaryFolder(new File("target")); + + private static AzureDataStore dataStore; + + @BeforeClass + public static void setupDataStore() throws Exception { + assumeTrue(isAzureConfigured() && !StringUtils.isEmpty(System.getProperty("test.opts.memory"))); + + dataStore = (AzureDataStore) getAzureDataStore(getAzureConfig(), homeDir.newFolder().getAbsolutePath()); + dataStore.setDirectDownloadURIExpirySeconds(expirySeconds); + dataStore.setDirectUploadURIExpirySeconds(expirySeconds); + } + + @Override + protected ConfigurableDataRecordAccessProvider getDataStore() { + return dataStore; + } + + @Override + protected DataRecord doGetRecord(DataStore ds, DataIdentifier identifier) throws DataStoreException { + return ds.getRecord(identifier); + } + + @Override + protected void doDeleteRecord(DataStore ds, DataIdentifier identifier) throws DataStoreException { + ((AzureDataStore)ds).deleteRecord(identifier); + } + + @Override + protected long getProviderMaxPartSize() { + return AzureConstantsV12.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; + } + + @Override + protected HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException { + HttpsURLConnection conn = (HttpsURLConnection) uri.toURL().openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("Content-Length", String.valueOf(length)); + conn.setRequestProperty("Date", DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssX") + .withZone(ZoneOffset.UTC) + .format(Instant.now())); + conn.setRequestProperty("x-ms-version", "2017-11-09"); + return conn; + } + + private static boolean isAzureConfigured() { + Properties props = getAzureConfig(); + if (!props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY) || !props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME) + || !(props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME))) { + if (!props.containsKey(AzureConstantsV12.AZURE_SAS) || !props.containsKey(AzureConstantsV12.AZURE_BLOB_ENDPOINT) + || !(props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME))) { + return props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME) && props.containsKey(AzureConstantsV12.AZURE_TENANT_ID) && + props.containsKey(AzureConstantsV12.AZURE_CLIENT_ID) && props.containsKey(AzureConstantsV12.AZURE_CLIENT_SECRET) && + props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME); + } + } + return true; + } + + private static Properties getAzureConfig() { + String config = System.getProperty(SYS_PROP_NAME); + if (StringUtils.isEmpty(config)) { + File cfgFile = new File(System.getProperty("user.home"), DEFAULT_PROPERTY_FILE); + if (cfgFile.exists()) { + config = cfgFile.getAbsolutePath(); + } + } + if (StringUtils.isEmpty(config)) { + config = DEFAULT_CONFIG_PATH; + } + + Properties props = new Properties(); + if (new File(config).exists()) { + InputStream is = null; + try { + is = new FileInputStream(config); + props.load(is); + } catch (Exception e) { + log.warn("Error loading azure config", e); + } finally { + IOUtils.closeQuietly(is); + } + props.putAll(DataStoreUtils.getConfig()); + Map filtered = MapUtils.filterEntries(MapUtils.fromProperties(props), + input -> !StringUtils.isEmpty(input.getValue())); + props = new Properties(); + props.putAll(filtered); + } + + props.setProperty(AzureConstants.AZURE_V12_ENABLED_PROPERTY, "true"); + return props; + } + + private static DataStore getAzureDataStore(Properties props, String homeDir) throws Exception { + AzureDataStore ds = new AzureDataStore(); + PropertiesUtil.populate(ds, MapUtils.fromProperties(props), false); + ds.setProperties(props); + ds.init(homeDir); + return ds; + } +} diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderV12Test.java similarity index 52% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderV12Test.java index 1e5243778b8..5b8ed7cdc41 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataRecordAccessProviderTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureDataRecordAccessProviderV12Test.java @@ -16,28 +16,38 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.Properties; import javax.net.ssl.HttpsURLConnection; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; import org.apache.jackrabbit.oak.spi.blob.data.DataStore; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions; +import org.apache.jackrabbit.oak.commons.PropertiesUtil; +import org.apache.jackrabbit.oak.commons.collections.MapUtils; import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.AbstractDataRecordAccessProviderTest; import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider; @@ -46,12 +56,30 @@ import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException; import org.apache.jackrabbit.oak.spi.blob.BlobOptions; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests direct access (presigned URL) functionality for Azure V12 backend. + * + *

Requires real Azure credentials because presigned download/upload URIs + * use HTTPS which Azurite does not support. Provide credentials via + * {@code -Dazure.config=} or place {@code azure.properties} in the + * user home directory.

+ */ +public class AzureDataRecordAccessProviderV12Test extends AbstractDataRecordAccessProviderTest { + + private static final Logger log = LoggerFactory.getLogger(AzureDataRecordAccessProviderV12Test.class); + + private static final String DEFAULT_CONFIG_PATH = "./src/test/resources/azure.properties"; + private static final String DEFAULT_PROPERTY_FILE = "azure.properties"; + private static final String SYS_PROP_NAME = "azure.config"; -public class AzureDataRecordAccessProviderTest extends AbstractDataRecordAccessProviderTest { @ClassRule public static TemporaryFolder homeDir = new TemporaryFolder(new File("target")); @@ -59,11 +87,11 @@ public class AzureDataRecordAccessProviderTest extends AbstractDataRecordAccessP @BeforeClass public static void setupDataStore() throws Exception { - dataStore = AzureDataStoreUtils.setupDirectAccessDataStore(homeDir, expirySeconds, expirySeconds); + dataStore = setupDirectAccessDataStore(homeDir, expirySeconds, expirySeconds); } private static AzureDataStore createDataStore(@NotNull Properties properties) throws Exception { - return AzureDataStoreUtils.setupDirectAccessDataStore(homeDir, expirySeconds, expirySeconds, properties); + return setupDirectAccessDataStore(homeDir, expirySeconds, expirySeconds, properties); } @Override @@ -73,7 +101,7 @@ protected ConfigurableDataRecordAccessProvider getDataStore() { @Override protected ConfigurableDataRecordAccessProvider getDataStore(@NotNull Properties overrideProperties) throws Exception { - return createDataStore(AzureDataStoreUtils.getDirectAccessDataStoreProperties(overrideProperties)); + return createDataStore(getDirectAccessDataStoreProperties(overrideProperties)); } @Override @@ -93,19 +121,19 @@ protected void doDeleteRecord(DataStore ds, DataIdentifier identifier) throws Da @Override protected long getProviderMinPartSize() { - return Math.max(0L, AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE); + return Math.max(0L, AzureConstantsV12.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE); } @Override protected long getProviderMaxPartSize() { - return AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; + return AzureConstantsV12.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE; } @Override - protected long getProviderMaxSinglePutSize() { return AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE; } + protected long getProviderMaxSinglePutSize() { return AzureConstantsV12.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE; } @Override - protected long getProviderMaxBinaryUploadSize() { return AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE; } + protected long getProviderMaxBinaryUploadSize() { return AzureConstantsV12.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE; } @Override protected boolean isSinglePutURI(URI uri) { @@ -117,7 +145,7 @@ protected boolean isSinglePutURI(URI uri) { @Override protected HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException { - return AzureDataStoreUtils.getHttpsConnection(length, uri); + return createHttpsConnection(length, uri); } @Test @@ -197,4 +225,113 @@ private static DataRecordDownloadOptions downloadOptionsWithMimeType(String mime ) ); } + + // --- Test utility methods (moved from AzureDataStoreUtilsV12) --- + + static boolean isAzureConfigured() { + Properties props = getAzureConfig(); + if (!props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY) || !props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME) + || !(props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME))) { + if (!props.containsKey(AzureConstantsV12.AZURE_SAS) || !props.containsKey(AzureConstantsV12.AZURE_BLOB_ENDPOINT) + || !(props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME))) { + return props.containsKey(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME) && props.containsKey(AzureConstantsV12.AZURE_TENANT_ID) && + props.containsKey(AzureConstantsV12.AZURE_CLIENT_ID) && props.containsKey(AzureConstantsV12.AZURE_CLIENT_SECRET) && + props.containsKey(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME); + } + } + return true; + } + + static Properties getAzureConfig() { + String config = System.getProperty(SYS_PROP_NAME); + if (StringUtils.isEmpty(config)) { + File cfgFile = new File(System.getProperty("user.home"), DEFAULT_PROPERTY_FILE); + if (cfgFile.exists()) { + config = cfgFile.getAbsolutePath(); + } + } + if (StringUtils.isEmpty(config)) { + config = DEFAULT_CONFIG_PATH; + } + + Properties props = new Properties(); + if (new File(config).exists()) { + InputStream is = null; + try { + is = new FileInputStream(config); + props.load(is); + } catch (Exception e) { + log.warn("Error loading azure config", e); + } finally { + IOUtils.closeQuietly(is); + } + props.putAll(DataStoreUtils.getConfig()); + Map filtered = MapUtils.filterEntries(MapUtils.fromProperties(props), + input -> !StringUtils.isEmpty(input.getValue())); + props = new Properties(); + props.putAll(filtered); + } + + props.setProperty(org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_V12_ENABLED_PROPERTY, "true"); + return props; + } + + static DataStore getAzureDataStore(Properties props, String homeDir) throws Exception { + AzureDataStore ds = new AzureDataStore(); + PropertiesUtil.populate(ds, MapUtils.fromProperties(props), false); + ds.setProperties(props); + ds.init(homeDir); + return ds; + } + + @SuppressWarnings("unchecked") + static T setupDirectAccessDataStore( + @NotNull final TemporaryFolder homeDir, + int directDownloadExpirySeconds, + int directUploadExpirySeconds) throws Exception { + return setupDirectAccessDataStore(homeDir, directDownloadExpirySeconds, directUploadExpirySeconds, null); + } + + @SuppressWarnings("unchecked") + static T setupDirectAccessDataStore( + @NotNull final TemporaryFolder homeDir, + int directDownloadExpirySeconds, + int directUploadExpirySeconds, + @Nullable final Properties overrideProperties) throws Exception { + assumeTrue(isAzureConfigured()); + T ds = (T) getAzureDataStore(getDirectAccessDataStoreProperties(overrideProperties), homeDir.newFolder().getAbsolutePath()); + if (ds instanceof ConfigurableDataRecordAccessProvider) { + ((ConfigurableDataRecordAccessProvider) ds).setDirectDownloadURIExpirySeconds(directDownloadExpirySeconds); + ((ConfigurableDataRecordAccessProvider) ds).setDirectUploadURIExpirySeconds(directUploadExpirySeconds); + } + return ds; + } + + static Properties getDirectAccessDataStoreProperties() { + return getDirectAccessDataStoreProperties(null); + } + + static Properties getDirectAccessDataStoreProperties(@Nullable final Properties overrideProperties) { + Properties mergedProperties = new Properties(); + mergedProperties.putAll(getAzureConfig()); + if (overrideProperties != null) { + mergedProperties.putAll(overrideProperties); + } + if (mergedProperties.getProperty("cacheSize", null) == null) { + mergedProperties.put("cacheSize", "0"); + } + return mergedProperties; + } + + static HttpsURLConnection createHttpsConnection(long length, URI uri) throws IOException { + HttpsURLConnection conn = (HttpsURLConnection) uri.toURL().openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("Content-Length", String.valueOf(length)); + conn.setRequestProperty("Date", DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssX") + .withZone(ZoneOffset.UTC) + .format(Instant.now())); + conn.setRequestProperty("x-ms-version", "2017-11-09"); + return conn; + } } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicyTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureHttpRequestLoggingPolicyV12Test.java similarity index 93% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicyTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureHttpRequestLoggingPolicyV12Test.java index 3aeb0724065..403745c9640 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureHttpRequestLoggingPolicyTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/AzureHttpRequestLoggingPolicyV12Test.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; @@ -36,7 +36,7 @@ import static org.junit.Assert.*; import static org.mockito.Mockito.*; -public class AzureHttpRequestLoggingPolicyTest { +public class AzureHttpRequestLoggingPolicyV12Test { private String originalVerboseProperty; @@ -58,7 +58,7 @@ public void tearDown() { @Test public void testLoggingPolicyCreation() { - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); assertNotNull("Logging policy should be created successfully", policy); } @@ -67,7 +67,7 @@ public void testProcessRequestWithVerboseDisabled() throws MalformedURLException // Ensure verbose logging is disabled System.clearProperty("blob.azure.v12.http.verbose.enabled"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); // Mock the context and request HttpPipelineCallContext context = mock(HttpPipelineCallContext.class); @@ -101,7 +101,7 @@ public void testProcessRequestWithVerboseEnabled() throws MalformedURLException // Enable verbose logging System.setProperty("blob.azure.v12.http.verbose.enabled", "true"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); // Mock the context and request HttpPipelineCallContext context = mock(HttpPipelineCallContext.class); @@ -136,7 +136,7 @@ public void testProcessRequestWithVerboseEnabled() throws MalformedURLException public void testProcessRequestWithDifferentHttpMethods() throws MalformedURLException { System.setProperty("blob.azure.v12.http.verbose.enabled", "true"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); // Test different HTTP methods HttpMethod[] methods = {HttpMethod.GET, HttpMethod.POST, HttpMethod.PUT, HttpMethod.DELETE, HttpMethod.HEAD}; @@ -171,7 +171,7 @@ public void testProcessRequestWithDifferentHttpMethods() throws MalformedURLExce public void testProcessRequestWithDifferentStatusCodes() throws MalformedURLException { System.setProperty("blob.azure.v12.http.verbose.enabled", "true"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); // Test different status codes int[] statusCodes = {200, 201, 204, 400, 404, 500}; @@ -206,7 +206,7 @@ public void testProcessRequestWithDifferentStatusCodes() throws MalformedURLExce public void testProcessRequestWithErrorInNextPolicy() throws MalformedURLException { System.setProperty("blob.azure.v12.http.verbose.enabled", "true"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); // Mock the context and request HttpPipelineCallContext context = mock(HttpPipelineCallContext.class); @@ -241,7 +241,7 @@ public void testProcessRequestWithVerbosePropertySetToFalse() throws MalformedUR // Explicitly set verbose logging to false System.setProperty("blob.azure.v12.http.verbose.enabled", "false"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); // Mock the context and request HttpPipelineCallContext context = mock(HttpPipelineCallContext.class); @@ -274,7 +274,7 @@ public void testProcessRequestWithVerbosePropertySetToFalse() throws MalformedUR public void testProcessRequestWithComplexUrl() throws MalformedURLException { System.setProperty("blob.azure.v12.http.verbose.enabled", "true"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); // Mock the context and request HttpPipelineCallContext context = mock(HttpPipelineCallContext.class); @@ -308,7 +308,7 @@ public void testProcessRequestWithComplexUrl() throws MalformedURLException { @Test public void testProcessRequestWithNullContext() { - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); HttpPipelineNextPolicy nextPolicy = mock(HttpPipelineNextPolicy.class); HttpResponse response = mock(HttpResponse.class); @@ -331,7 +331,7 @@ public void testProcessRequestWithNullContext() { @Test public void testProcessRequestWithNullNextPolicy() { - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); HttpPipelineCallContext context = mock(HttpPipelineCallContext.class); HttpRequest request = mock(HttpRequest.class); @@ -351,7 +351,7 @@ public void testProcessRequestWithNullNextPolicy() { public void testProcessRequestWithSlowResponse() throws MalformedURLException { System.setProperty("blob.azure.v12.http.verbose.enabled", "true"); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); HttpPipelineCallContext context = mock(HttpPipelineCallContext.class); HttpRequest request = mock(HttpRequest.class); @@ -383,7 +383,7 @@ public void testVerboseLoggingSystemPropertyDetection() { for (String value : testValues) { System.setProperty("blob.azure.v12.http.verbose.enabled", value); - AzureHttpRequestLoggingPolicy policy = new AzureHttpRequestLoggingPolicy(); + AzureHttpRequestLoggingPolicyV12 policy = new AzureHttpRequestLoggingPolicyV12(); assertNotNull("Policy should be created regardless of system property value", policy); } } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDSV12.java similarity index 51% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDSV12.java index 47c704d842d..8b2286eb6be 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDSV12.java @@ -16,45 +16,43 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; -import static org.junit.Assume.assumeTrue; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import org.apache.jackrabbit.oak.spi.blob.data.DataStore; -import org.apache.jackrabbit.oak.commons.junit.LogCustomizer; import org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreTest; import org.junit.After; -import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; -import org.slf4j.event.Level; +import org.junit.ClassRule; import java.util.Properties; /** - * Test {@link AzureDataStore} with AzureDataStore and local cache on. - * It requires to pass azure config file via system property or system properties by prefixing with 'ds.'. - * See details @ {@link AzureDataStoreUtils}. - * For e.g. -Dconfig=/opt/cq/azure.properties. Sample azure properties located at - * src/test/resources/azure.properties + * Test {@link AzureDataStore} with AzureBlobStoreBackendV12 and local cache on. + * Uses Azurite (local Azure emulator) via Docker for testing. */ -public class TestAzureDS extends AbstractDataStoreTest { +public class TestAzureDSV12 extends AbstractDataStoreTest { + + @ClassRule + public static AzuriteDockerRule azurite = new AzuriteDockerRule(); protected Properties props = new Properties(); protected String container; - @BeforeClass - public static void assumptions() { - assumeTrue(AzureDataStoreUtils.isAzureConfigured()); - } - @Override @Before public void setUp() throws Exception { - props.putAll(AzureDataStoreUtils.getAzureConfig()); + props.setProperty(AzureConstantsV12.AZURE_CONNECTION_STRING, azurite.getConnectionString()); + props.setProperty(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); + props.setProperty(AzureConstantsV12.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); + props.setProperty(AzureConstants.AZURE_V12_ENABLED_PROPERTY, "true"); container = randomGen.nextInt(9999) + "-" + randomGen.nextInt(9999) + "-test"; - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, container); + props.setProperty(AzureConstantsV12.AZURE_BLOB_CONTAINER_NAME, container); + props.setProperty(AzureConstantsV12.AZURE_CREATE_CONTAINER, "true"); props.setProperty("secret", "123456"); super.setUp(); } @@ -63,15 +61,9 @@ public void setUp() throws Exception { @After public void tearDown() { try { - LogCustomizer customizer = LogCustomizer.forLogger(AzureBlobContainerProvider.class.getName()) - .filter(Level.INFO) - .create(); - customizer.starting(); super.tearDown(); - Assert.assertEquals(1, customizer.getLogs().size()); - Assert.assertEquals("Refresh token executor service shutdown completed", customizer.getLogs().get(0)); - customizer.finished(); - AzureDataStoreUtils.deleteContainer(container); + // Clean up test container in Azurite + azurite.getContainer(container, azurite.getConnectionString()).deleteIfExists(); } catch (Exception ignore) { } @@ -81,7 +73,12 @@ public void tearDown() { protected DataStore createDataStore() { DataStore azureds = null; try { - azureds = AzureDataStoreUtils.getAzureDataStore(props, dataStoreDir); + AzureDataStore ds = new AzureDataStore(); + org.apache.jackrabbit.oak.commons.PropertiesUtil.populate( + ds, org.apache.jackrabbit.oak.commons.collections.MapUtils.fromProperties(props), false); + ds.setProperties(props); + ds.init(dataStoreDir); + azureds = ds; } catch (Exception e) { e.printStackTrace(); } @@ -89,7 +86,7 @@ protected DataStore createDataStore() { return azureds; } - /**---------- Skipped -----------**/ + /**---------- Skipped (not supported by CachingDataStore on Azurite) -----------**/ @Override public void testUpdateLastModifiedOnAccess() { } @@ -97,4 +94,8 @@ public void testUpdateLastModifiedOnAccess() { @Override public void testDeleteAllOlderThan() { } + + @Override + public void testDeleteRecord() { + } } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDSWithSmallCacheV12.java similarity index 68% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDSWithSmallCacheV12.java index c03b869a4e0..f97a32add3d 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDSWithSmallCacheV12.java @@ -14,22 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; import org.apache.jackrabbit.oak.spi.blob.data.CachingDataStore; import org.apache.jackrabbit.oak.spi.blob.data.LocalCache; import org.junit.Before; /** - * Test {@link CachingDataStore} with AzureBlobStoreBackend and with very small size (@link - * {@link LocalCache}. - * It requires to pass azure config file via system property or system properties by prefixing with 'ds.'. - * See details @ {@link AzureDataStoreUtils}. - * For e.g. -Dconfig=/opt/cq/azure.properties. Sample azure properties located at - * src/test/resources/azure.properties - + * Test {@link CachingDataStore} with AzureBlobStoreBackendV12 and with very small size + * {@link LocalCache}. Uses Azurite (local Azure emulator) via Docker for testing. */ -public class TestAzureDSWithSmallCache extends TestAzureDS { +public class TestAzureDSWithSmallCacheV12 extends TestAzureDSV12 { @Override @Before diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDsCacheOffV12.java similarity index 68% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDsCacheOffV12.java index 01efe76648b..88860cbfed2 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/TestAzureDsCacheOffV12.java @@ -14,21 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; import org.apache.jackrabbit.oak.spi.blob.data.CachingDataStore; import org.junit.Before; /** - * Test {@link CachingDataStore} with AzureBlobStoreBackend - * and local cache Off. - * It requires to pass azure config file via system property or system properties by prefixing with 'ds.'. - * See details @ {@link AzureDataStoreUtils}. - * For e.g. -Dconfig=/opt/cq/azure.properties. Sample azure properties located at - * src/test/resources/azure.properties - + * Test {@link CachingDataStore} with AzureBlobStoreBackendV12 + * and local cache Off. Uses Azurite (local Azure emulator) via Docker for testing. */ -public class TestAzureDsCacheOff extends TestAzureDS { +public class TestAzureDsCacheOffV12 extends TestAzureDSV12 { @Override @Before diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/UtilsV12Test.java similarity index 68% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/UtilsV12Test.java index b33d14988e0..269a4a6bd38 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/UtilsTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v12/UtilsV12Test.java @@ -14,7 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage; +package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v12; + +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.common.policy.RequestRetryOptions; @@ -35,7 +37,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; -public class UtilsTest { +public class UtilsV12Test { @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -43,17 +45,17 @@ public class UtilsTest { @Test public void testConnectionStringIsBasedOnProperty() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_CONNECTION_STRING, "DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey"); - String connectionString = Utils.getConnectionStringFromProperties(properties); + properties.put(AzureConstantsV12.AZURE_CONNECTION_STRING, "DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey"); + String connectionString = UtilsV12.getConnectionStringFromProperties(properties); assertEquals("DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey", connectionString); } @Test public void testConnectionStringIsBasedOnSAS() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_SAS, "sas"); - properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint"); - String connectionString = Utils.getConnectionStringFromProperties(properties); + properties.put(AzureConstantsV12.AZURE_SAS, "sas"); + properties.put(AzureConstantsV12.AZURE_BLOB_ENDPOINT, "endpoint"); + String connectionString = UtilsV12.getConnectionStringFromProperties(properties); assertEquals(connectionString, String.format("BlobEndpoint=%s;SharedAccessSignature=%s", "endpoint", "sas")); } @@ -61,9 +63,9 @@ public void testConnectionStringIsBasedOnSAS() { @Test public void testConnectionStringIsBasedOnSASWithoutEndpoint() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_SAS, "sas"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "account"); - String connectionString = Utils.getConnectionStringFromProperties(properties); + properties.put(AzureConstantsV12.AZURE_SAS, "sas"); + properties.put(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, "account"); + String connectionString = UtilsV12.getConnectionStringFromProperties(properties); assertEquals(connectionString, String.format("AccountName=%s;SharedAccessSignature=%s", "account", "sas")); } @@ -71,10 +73,10 @@ public void testConnectionStringIsBasedOnSASWithoutEndpoint() { @Test public void testConnectionStringIsBasedOnAccessKeyIfSASMissing() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); + properties.put(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); + properties.put(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); - String connectionString = Utils.getConnectionStringFromProperties(properties); + String connectionString = UtilsV12.getConnectionStringFromProperties(properties); assertEquals(connectionString, String.format("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s","accessKey","secretKey")); } @@ -82,13 +84,13 @@ public void testConnectionStringIsBasedOnAccessKeyIfSASMissing() { @Test public void testConnectionStringSASIsPriority() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_SAS, "sas"); - properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint"); + properties.put(AzureConstantsV12.AZURE_SAS, "sas"); + properties.put(AzureConstantsV12.AZURE_BLOB_ENDPOINT, "endpoint"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); + properties.put(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); + properties.put(AzureConstantsV12.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); - String connectionString = Utils.getConnectionStringFromProperties(properties); + String connectionString = UtilsV12.getConnectionStringFromProperties(properties); assertEquals(connectionString, String.format("BlobEndpoint=%s;SharedAccessSignature=%s", "endpoint", "sas")); } @@ -101,14 +103,14 @@ public void testReadConfig() throws IOException { writer.write("key2=value2\n"); } - Properties properties = Utils.readConfig(tempFile.getAbsolutePath()); + Properties properties = UtilsV12.readConfig(tempFile.getAbsolutePath()); assertEquals("value1", properties.getProperty("key1")); assertEquals("value2", properties.getProperty("key2")); } @Test public void testReadConfig_exception() { - assertThrows(IOException.class, () -> Utils.readConfig("non-existent-file")); + assertThrows(IOException.class, () -> UtilsV12.readConfig("non-existent-file")); } @Test @@ -122,31 +124,31 @@ public void testGetBlobContainer() throws IOException, DataStoreException { Properties properties = new Properties(); properties.load(new FileInputStream(tempFile)); - String connectionString = Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, "http://127.0.0.1:10000/devstoreaccount1" ); + String connectionString = UtilsV12.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, "http://127.0.0.1:10000/devstoreaccount1" ); String containerName = "test-container"; - RequestRetryOptions retryOptions = Utils.getRetryOptions("3", 3, null); + RequestRetryOptions retryOptions = UtilsV12.getRetryOptions("3", 3, null); - BlobContainerClient containerClient = Utils.getBlobContainer(connectionString, containerName, retryOptions, properties); + BlobContainerClient containerClient = UtilsV12.getBlobContainer(connectionString, containerName, retryOptions, properties); assertNotNull(containerClient); } @Test public void testGetRetryOptions() { - RequestRetryOptions retryOptions = Utils.getRetryOptions("3", 3, null); + RequestRetryOptions retryOptions = UtilsV12.getRetryOptions("3", 3, null); assertNotNull(retryOptions); assertEquals(3, retryOptions.getMaxTries()); } @Test public void testGetRetryOptionsNoRetry() { - RequestRetryOptions retryOptions = Utils.getRetryOptions("0",3, null); + RequestRetryOptions retryOptions = UtilsV12.getRetryOptions("0",3, null); assertNotNull(retryOptions); assertEquals(1, retryOptions.getMaxTries()); } @Test public void testGetRetryOptionsInvalid() { - RequestRetryOptions retryOptions = Utils.getRetryOptions("-1", 3, null); + RequestRetryOptions retryOptions = UtilsV12.getRetryOptions("-1", 3, null); assertNull(retryOptions); } @@ -156,7 +158,7 @@ public void testGetConnectionString() { String accountKey = "testkey"; String blobEndpoint = "https://testaccount.blob.core.windows.net"; - String connectionString = Utils.getConnectionString(accountName, accountKey, blobEndpoint); + String connectionString = UtilsV12.getConnectionString(accountName, accountKey, blobEndpoint); String expected = "DefaultEndpointsProtocol=https;AccountName=testaccount;AccountKey=testkey;BlobEndpoint=https://testaccount.blob.core.windows.net"; assertEquals("Connection string should match expected format", expected, connectionString); } @@ -166,7 +168,7 @@ public void testGetConnectionStringWithoutEndpoint() { String accountName = "testaccount"; String accountKey = "testkey"; - String connectionString = Utils.getConnectionString(accountName, accountKey, null); + String connectionString = UtilsV12.getConnectionString(accountName, accountKey, null); String expected = "DefaultEndpointsProtocol=https;AccountName=testaccount;AccountKey=testkey"; assertEquals("Connection string should match expected format without endpoint", expected, connectionString); } @@ -176,7 +178,7 @@ public void testGetConnectionStringWithEmptyEndpoint() { String accountName = "testaccount"; String accountKey = "testkey"; - String connectionString = Utils.getConnectionString(accountName, accountKey, ""); + String connectionString = UtilsV12.getConnectionString(accountName, accountKey, ""); String expected = "DefaultEndpointsProtocol=https;AccountName=testaccount;AccountKey=testkey"; assertEquals("Connection string should match expected format with empty endpoint", expected, connectionString); } @@ -187,7 +189,7 @@ public void testGetConnectionStringForSas() { String blobEndpoint = "https://testaccount.blob.core.windows.net"; String accountName = "testaccount"; - String connectionString = Utils.getConnectionStringForSas(sasUri, blobEndpoint, accountName); + String connectionString = UtilsV12.getConnectionStringForSas(sasUri, blobEndpoint, accountName); String expected = "BlobEndpoint=https://testaccount.blob.core.windows.net;SharedAccessSignature=sas-token"; assertEquals("SAS connection string should match expected format", expected, connectionString); } @@ -197,7 +199,7 @@ public void testGetConnectionStringForSasWithoutEndpoint() { String sasUri = "sas-token"; String accountName = "testaccount"; - String connectionString = Utils.getConnectionStringForSas(sasUri, null, accountName); + String connectionString = UtilsV12.getConnectionStringForSas(sasUri, null, accountName); String expected = "AccountName=testaccount;SharedAccessSignature=sas-token"; assertEquals("SAS connection string should use account name when endpoint is null", expected, connectionString); } @@ -207,7 +209,7 @@ public void testGetConnectionStringForSasWithEmptyEndpoint() { String sasUri = "sas-token"; String accountName = "testaccount"; - String connectionString = Utils.getConnectionStringForSas(sasUri, "", accountName); + String connectionString = UtilsV12.getConnectionStringForSas(sasUri, "", accountName); String expected = "AccountName=testaccount;SharedAccessSignature=sas-token"; assertEquals("SAS connection string should use account name when endpoint is empty", expected, connectionString); } @@ -215,10 +217,10 @@ public void testGetConnectionStringForSasWithEmptyEndpoint() { @Test public void testComputeProxyOptionsWithBothHostAndPort() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, "proxy.example.com"); - properties.setProperty(AzureConstants.PROXY_PORT, "8080"); + properties.setProperty(AzureConstantsV12.PROXY_HOST, "proxy.example.com"); + properties.setProperty(AzureConstantsV12.PROXY_PORT, "8080"); - com.azure.core.http.ProxyOptions proxyOptions = Utils.computeProxyOptions(properties); + com.azure.core.http.ProxyOptions proxyOptions = UtilsV12.computeProxyOptions(properties); assertNotNull("Proxy options should not be null", proxyOptions); assertEquals("Proxy host should match", "proxy.example.com", proxyOptions.getAddress().getHostName()); assertEquals("Proxy port should match", 8080, proxyOptions.getAddress().getPort()); @@ -227,28 +229,28 @@ public void testComputeProxyOptionsWithBothHostAndPort() { @Test(expected = NumberFormatException.class) public void testComputeProxyOptionsWithInvalidPort() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, "proxy.example.com"); - properties.setProperty(AzureConstants.PROXY_PORT, "invalid"); + properties.setProperty(AzureConstantsV12.PROXY_HOST, "proxy.example.com"); + properties.setProperty(AzureConstantsV12.PROXY_PORT, "invalid"); - Utils.computeProxyOptions(properties); + UtilsV12.computeProxyOptions(properties); fail("Expected NumberFormatException when port is invalid"); } @Test public void testComputeProxyOptionsWithHostOnly() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, "proxy.example.com"); + properties.setProperty(AzureConstantsV12.PROXY_HOST, "proxy.example.com"); - com.azure.core.http.ProxyOptions proxyOptions = Utils.computeProxyOptions(properties); + com.azure.core.http.ProxyOptions proxyOptions = UtilsV12.computeProxyOptions(properties); assertNull("Proxy options should be null when port is missing", proxyOptions); } @Test public void testComputeProxyOptionsWithPortOnly() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_PORT, "8080"); + properties.setProperty(AzureConstantsV12.PROXY_PORT, "8080"); - com.azure.core.http.ProxyOptions proxyOptions = Utils.computeProxyOptions(properties); + com.azure.core.http.ProxyOptions proxyOptions = UtilsV12.computeProxyOptions(properties); assertNull("Proxy options should be null when host is missing", proxyOptions); } @@ -256,17 +258,17 @@ public void testComputeProxyOptionsWithPortOnly() { public void testComputeProxyOptionsWithEmptyProperties() { Properties properties = new Properties(); - com.azure.core.http.ProxyOptions proxyOptions = Utils.computeProxyOptions(properties); + com.azure.core.http.ProxyOptions proxyOptions = UtilsV12.computeProxyOptions(properties); assertNull("Proxy options should be null with empty properties", proxyOptions); } @Test public void testComputeProxyOptionsWithEmptyValues() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, ""); - properties.setProperty(AzureConstants.PROXY_PORT, ""); + properties.setProperty(AzureConstantsV12.PROXY_HOST, ""); + properties.setProperty(AzureConstantsV12.PROXY_PORT, ""); - com.azure.core.http.ProxyOptions proxyOptions = Utils.computeProxyOptions(properties); + com.azure.core.http.ProxyOptions proxyOptions = UtilsV12.computeProxyOptions(properties); assertNull("Proxy options should be null with empty values", proxyOptions); } @@ -275,7 +277,7 @@ public void testGetBlobContainerFromConnectionString() { String connectionString = getConnectionString(); String containerName = "test-container"; - BlobContainerClient containerClient = Utils.getBlobContainerFromConnectionString(connectionString, containerName); + BlobContainerClient containerClient = UtilsV12.getBlobContainerFromConnectionString(connectionString, containerName); assertNotNull("Container client should not be null", containerClient); assertEquals("Container name should match", containerName, containerClient.getBlobContainerName()); } @@ -283,14 +285,14 @@ public void testGetBlobContainerFromConnectionString() { @Test public void testGetRetryOptionsWithSecondaryLocation() { String secondaryLocation = "https://testaccount-secondary.blob.core.windows.net"; - RequestRetryOptions retryOptions = Utils.getRetryOptions("3", 30, secondaryLocation); + RequestRetryOptions retryOptions = UtilsV12.getRetryOptions("3", 30, secondaryLocation); assertNotNull("Retry options should not be null", retryOptions); assertEquals("Max tries should be 3", 3, retryOptions.getMaxTries()); } @Test public void testGetRetryOptionsWithNullValues() { - RequestRetryOptions retryOptions = Utils.getRetryOptions(null, null, null); + RequestRetryOptions retryOptions = UtilsV12.getRetryOptions(null, null, null); assertNull("Retry options should be null with null max retry count", retryOptions); } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8BuilderTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8BuilderTest.java index 9a8cac2f9cf..3478b93a123 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8BuilderTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8BuilderTest.java @@ -18,7 +18,6 @@ */ package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.junit.After; import org.junit.Test; @@ -54,14 +53,14 @@ public void tearDown() { @Test public void testBuilderInitializeWithProperties() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, CONNECTION_STRING); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ACCOUNT_NAME); - properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, BLOB_ENDPOINT); - properties.setProperty(AzureConstants.AZURE_SAS, SAS_TOKEN); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, ACCOUNT_KEY); - properties.setProperty(AzureConstants.AZURE_TENANT_ID, TENANT_ID); - properties.setProperty(AzureConstants.AZURE_CLIENT_ID, CLIENT_ID); - properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, CLIENT_SECRET); + properties.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, CONNECTION_STRING); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, ACCOUNT_NAME); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, BLOB_ENDPOINT); + properties.setProperty(AzureConstantsV8.AZURE_SAS, SAS_TOKEN); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, ACCOUNT_KEY); + properties.setProperty(AzureConstantsV8.AZURE_TENANT_ID, TENANT_ID); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_ID, CLIENT_ID); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, CLIENT_SECRET); provider = AzureBlobContainerProviderV8.Builder .builder(CONTAINER_NAME) @@ -199,8 +198,8 @@ public void testBuilderWithPartialConfiguration() { public void testBuilderWithPropertiesOverride() { // Test that explicit builder methods override properties Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "properties-account"); - properties.setProperty(AzureConstants.AZURE_TENANT_ID, "properties-tenant"); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "properties-account"); + properties.setProperty(AzureConstantsV8.AZURE_TENANT_ID, "properties-tenant"); provider = AzureBlobContainerProviderV8.Builder .builder(CONTAINER_NAME) diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8ContainerOperationsTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8ContainerOperationsIT.java similarity index 99% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8ContainerOperationsTest.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8ContainerOperationsIT.java index 567d8adddc5..c7200ae7dbe 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8ContainerOperationsTest.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8ContainerOperationsIT.java @@ -36,7 +36,7 @@ * Tests getBlobContainer operations and container access patterns. * Includes both unit tests with mock credentials and integration tests with Azurite. */ -public class AzureBlobContainerProviderV8ContainerOperationsTest { +public class AzureBlobContainerProviderV8ContainerOperationsIT { @ClassRule public static AzuriteDockerRule azurite = new AzuriteDockerRule(); @@ -369,4 +369,4 @@ public void testGetBlobContainerWithAzuriteVerifyContainerExists() throws Except assertEquals("Container name should match", CONTAINER_NAME, container.getName()); assertNotNull("Container URI should not be null", container.getUri()); } -} \ No newline at end of file +} diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8Test.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8IT.java similarity index 94% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8Test.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8IT.java index 0249636de0b..d917b8d50d9 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8Test.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobContainerProviderV8IT.java @@ -26,7 +26,6 @@ import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import org.jetbrains.annotations.NotNull; import org.junit.After; @@ -56,7 +55,7 @@ import static org.junit.Assert.*; import static org.junit.Assume.assumeNotNull; -public class AzureBlobContainerProviderV8Test { +public class AzureBlobContainerProviderV8IT { private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID"; @@ -89,14 +88,14 @@ public void tearDown() throws Exception { @Test public void testBuilderWithAllProperties() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, "test-connection"); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "testaccount"); - properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "https://test.blob.core.windows.net"); - properties.setProperty(AzureConstants.AZURE_SAS, "test-sas"); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "test-key"); - properties.setProperty(AzureConstants.AZURE_TENANT_ID, "test-tenant"); - properties.setProperty(AzureConstants.AZURE_CLIENT_ID, "test-client"); - properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, "test-secret"); + properties.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, "test-connection"); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "testaccount"); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "https://test.blob.core.windows.net"); + properties.setProperty(AzureConstantsV8.AZURE_SAS, "test-sas"); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "test-key"); + properties.setProperty(AzureConstantsV8.AZURE_TENANT_ID, "test-tenant"); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_ID, "test-client"); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, "test-secret"); AzureBlobContainerProviderV8 provider = AzureBlobContainerProviderV8.Builder .builder(CONTAINER_NAME) @@ -142,8 +141,8 @@ public void testBuilderWithEmptyProperties() { public void testBuilderWithNullProperties() { Properties properties = new Properties(); // Properties with null values should default to empty strings - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, ""); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); + properties.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, ""); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, ""); AzureBlobContainerProviderV8 provider = AzureBlobContainerProviderV8.Builder .builder(CONTAINER_NAME) @@ -477,11 +476,11 @@ private Properties getPropertiesWithServicePrincipals() { final String clientSecret = getEnvironmentVariable(AZURE_CLIENT_SECRET); Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, accountName); - properties.setProperty(AzureConstants.AZURE_TENANT_ID, tenantId); - properties.setProperty(AzureConstants.AZURE_CLIENT_ID, clientId); - properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, clientSecret); - properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, accountName); + properties.setProperty(AzureConstantsV8.AZURE_TENANT_ID, tenantId); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_ID, clientId); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, clientSecret); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); return properties; } @@ -599,32 +598,32 @@ private CloudBlobContainer createBlobContainer() throws Exception { private static Properties getConfigurationWithSasToken(String sasToken) { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_SAS, sasToken); - properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "false"); - properties.setProperty(AzureConstants.AZURE_REF_ON_INIT, "false"); + properties.setProperty(AzureConstantsV8.AZURE_SAS, sasToken); + properties.setProperty(AzureConstantsV8.AZURE_CREATE_CONTAINER, "false"); + properties.setProperty(AzureConstantsV8.AZURE_REF_ON_INIT, "false"); return properties; } private static Properties getConfigurationWithAccessKey() { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, AzuriteDockerRule.ACCOUNT_KEY); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, AzuriteDockerRule.ACCOUNT_KEY); return properties; } @NotNull private static Properties getConfigurationWithConnectionString() { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); + properties.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, getConnectionString()); return properties; } @NotNull private static Properties getBasicConfiguration() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); - properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); - properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, ""); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); + properties.setProperty(AzureConstantsV8.AZURE_CREATE_CONTAINER, ""); return properties; } @@ -902,4 +901,4 @@ private static void assertReferenceSecret(AzureBlobStoreBackendV8 azureBlobStore assertTrue("reference key is empty", refRec.getLength() > 0); } -} \ No newline at end of file +} diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8Test.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8IT.java similarity index 85% rename from oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8Test.java rename to oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8IT.java index 5447776277f..aff94a26b21 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8Test.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8IT.java @@ -26,7 +26,6 @@ import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import org.jetbrains.annotations.NotNull; import org.junit.After; @@ -58,9 +57,9 @@ import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ; import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE; import static java.util.stream.Collectors.toSet; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT; -import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BlOB_META_DIR_NAME; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureConstantsV8.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT; +import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureConstantsV8.AZURE_BLOB_META_DIR_NAME; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -70,7 +69,7 @@ import static org.junit.Assert.fail; import static org.junit.Assume.assumeNotNull; -public class AzureBlobStoreBackendV8Test { +public class AzureBlobStoreBackendV8IT { private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID"; private static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID"; @@ -263,11 +262,11 @@ private Properties getPropertiesWithServicePrincipals() { final String clientSecret = getEnvironmentVariable(AZURE_CLIENT_SECRET); Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, accountName); - properties.setProperty(AzureConstants.AZURE_TENANT_ID, tenantId); - properties.setProperty(AzureConstants.AZURE_CLIENT_ID, clientId); - properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, clientSecret); - properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, accountName); + properties.setProperty(AzureConstantsV8.AZURE_TENANT_ID, tenantId); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_ID, clientId); + properties.setProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, clientSecret); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); return properties; } @@ -285,32 +284,32 @@ private CloudBlobContainer createBlobContainer() throws Exception { private static Properties getConfigurationWithSasToken(String sasToken) { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_SAS, sasToken); - properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "false"); - properties.setProperty(AzureConstants.AZURE_REF_ON_INIT, "false"); + properties.setProperty(AzureConstantsV8.AZURE_SAS, sasToken); + properties.setProperty(AzureConstantsV8.AZURE_CREATE_CONTAINER, "false"); + properties.setProperty(AzureConstantsV8.AZURE_REF_ON_INIT, "false"); return properties; } private static Properties getConfigurationWithAccessKey() { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, AzuriteDockerRule.ACCOUNT_KEY); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, AzuriteDockerRule.ACCOUNT_KEY); return properties; } @NotNull private static Properties getConfigurationWithConnectionString() { Properties properties = getBasicConfiguration(); - properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); + properties.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, getConnectionString()); return properties; } @NotNull private static Properties getBasicConfiguration() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); - properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); - properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); - properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, ""); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME); + properties.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); + properties.setProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); + properties.setProperty(AzureConstantsV8.AZURE_CREATE_CONTAINER, ""); return properties; } @@ -473,7 +472,7 @@ public void testMetadataDirectoryStructureV8() throws Exception { // In V8, metadata is stored in a directory structure com.microsoft.azure.storage.blob.CloudBlobDirectory metaDir = - azureContainer.getDirectoryReference(AZURE_BlOB_META_DIR_NAME); + azureContainer.getDirectoryReference(AZURE_BLOB_META_DIR_NAME); com.microsoft.azure.storage.blob.CloudBlockBlob blob = metaDir.getBlockBlobReference(testMetadataName); assertTrue("Blob should exist at expected path in V8", blob.exists()); @@ -549,8 +548,8 @@ public void testInitWithNullPropertiesAndValidConfigFile() throws Exception { public void testInitWithInvalidConnectionString() { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = new Properties(); - props.setProperty(AzureConstants.AZURE_CONNECTION_STRING, "invalid-connection-string"); - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, "test-container"); + props.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, "invalid-connection-string"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, "test-container"); backend.setProperties(props); try { @@ -571,7 +570,7 @@ public void testConcurrentRequestCountValidation() throws Exception { // Test with too low concurrent request count AzureBlobStoreBackendV8 backend1 = new AzureBlobStoreBackendV8(); Properties props1 = getConfigurationWithConnectionString(); - props1.setProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1"); // Too low + props1.setProperty(AzureConstantsV8.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1"); // Too low backend1.setProperties(props1); backend1.init(); // Should reset to default minimum @@ -581,7 +580,7 @@ public void testConcurrentRequestCountValidation() throws Exception { // Test with too high concurrent request count AzureBlobStoreBackendV8 backend2 = new AzureBlobStoreBackendV8(); Properties props2 = getConfigurationWithConnectionString(); - props2.setProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "100"); // Too high + props2.setProperty(AzureConstantsV8.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "100"); // Too high backend2.setProperties(props2); backend2.init(); // Should reset to default maximum @@ -651,7 +650,7 @@ public void testNullParameterValidation() throws Exception { backend.read(null); fail("Expected NullPointerException for null identifier in read"); } catch (NullPointerException e) { - assertEquals("identifier must not be null", e.getMessage()); + assertEquals("identifier", e.getMessage()); } // Test null identifier in getRecord @@ -659,7 +658,7 @@ public void testNullParameterValidation() throws Exception { backend.getRecord(null); fail("Expected NullPointerException for null identifier in getRecord"); } catch (NullPointerException e) { - assertEquals("identifier must not be null", e.getMessage()); + assertEquals("identifier", e.getMessage()); } // Test null identifier in deleteRecord @@ -667,7 +666,7 @@ public void testNullParameterValidation() throws Exception { backend.deleteRecord(null); fail("Expected NullPointerException for null identifier in deleteRecord"); } catch (NullPointerException e) { - assertEquals("identifier must not be null", e.getMessage()); + assertEquals("identifier", e.getMessage()); } // Test null input in addMetadataRecord @@ -675,7 +674,7 @@ public void testNullParameterValidation() throws Exception { backend.addMetadataRecord((java.io.InputStream) null, "test"); fail("Expected NullPointerException for null input in addMetadataRecord"); } catch (NullPointerException e) { - assertEquals("input must not be null", e.getMessage()); + assertEquals("input", e.getMessage()); } // Test null name in addMetadataRecord @@ -683,7 +682,7 @@ public void testNullParameterValidation() throws Exception { backend.addMetadataRecord(new ByteArrayInputStream("test".getBytes()), null); fail("Expected IllegalArgumentException for null name in addMetadataRecord"); } catch (IllegalArgumentException e) { - assertEquals("name should not be empty", e.getMessage()); + assertEquals("name", e.getMessage()); } } @@ -742,7 +741,7 @@ public void testDeleteAllMetadataRecordsWithNullPrefix() throws Exception { backend.deleteAllMetadataRecords(null); fail("Expected NullPointerException for null prefix"); } catch (NullPointerException e) { - assertEquals("prefix must not be null", e.getMessage()); + assertEquals("prefix", e.getMessage()); } } @@ -758,7 +757,7 @@ public void testGetAllMetadataRecordsWithNullPrefix() throws Exception { backend.getAllMetadataRecords(null); fail("Expected NullPointerException for null prefix"); } catch (NullPointerException e) { - assertEquals("prefix must not be null", e.getMessage()); + assertEquals("prefix", e.getMessage()); } } @@ -787,7 +786,7 @@ public void testWriteWithNullFile() throws Exception { backend.write(new DataIdentifier("test"), null); fail("Expected NullPointerException for null file"); } catch (NullPointerException e) { - assertEquals("file must not be null", e.getMessage()); + assertEquals("file", e.getMessage()); } } @@ -804,7 +803,7 @@ public void testWriteWithNullIdentifier() throws Exception { backend.write(null, tempFile); fail("Expected NullPointerException for null identifier"); } catch (NullPointerException e) { - assertEquals("identifier must not be null", e.getMessage()); + assertEquals("identifier", e.getMessage()); } finally { tempFile.delete(); } @@ -856,7 +855,7 @@ public void testAddMetadataRecordWithNullFile() throws Exception { backend.addMetadataRecord((java.io.File) null, "test"); fail("Expected NullPointerException for null file"); } catch (NullPointerException e) { - assertEquals("input must not be null", e.getMessage()); + assertEquals("input", e.getMessage()); } } @@ -1253,9 +1252,9 @@ public void testHttpDownloadURIConfiguration() throws Exception { Properties props = getConfigurationWithConnectionString(); // Configure download URI settings - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "100"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, "custom.domain.com"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "100"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, "custom.domain.com"); backend.setProperties(props); backend.init(); @@ -1272,8 +1271,8 @@ public void testHttpUploadURIConfiguration() throws Exception { Properties props = getConfigurationWithConnectionString(); // Configure upload URI settings - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, "upload.domain.com"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, "upload.domain.com"); backend.setProperties(props); backend.init(); @@ -1291,7 +1290,7 @@ public void testSecondaryLocationConfiguration() { Properties props = getConfigurationWithConnectionString(); // Enable secondary location - props.setProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME, "true"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME, "true"); backend.setProperties(props); @@ -1321,7 +1320,7 @@ public void testRequestTimeoutConfiguration() throws Exception { Properties props = getConfigurationWithConnectionString(); // Set request timeout - props.setProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT, "30000"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_REQUEST_TIMEOUT, "30000"); backend.setProperties(props); backend.init(); @@ -1342,7 +1341,7 @@ public void testPresignedDownloadURIVerifyExistsConfiguration() throws Exception Properties props = getConfigurationWithConnectionString(); // Disable verify exists for presigned download URIs - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "false"); backend.setProperties(props); backend.init(); @@ -1360,7 +1359,7 @@ public void testCreateContainerConfiguration() throws Exception { Properties props = getConfigurationWithConnectionString(); // Disable container creation - props.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "false"); + props.setProperty(AzureConstantsV8.AZURE_CREATE_CONTAINER, "false"); backend.setProperties(props); backend.init(); @@ -1376,8 +1375,8 @@ public void testIteratorWithEmptyContainer() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getBasicConfiguration(); - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, "empty-container"); - props.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, "empty-container"); + props.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, getConnectionString()); backend.setProperties(props); backend.init(); @@ -1469,7 +1468,7 @@ public void testAddMetadataRecordWithEmptyName() throws Exception { backend.addMetadataRecord(new ByteArrayInputStream("test".getBytes()), ""); fail("Expected IllegalArgumentException for empty name"); } catch (IllegalArgumentException e) { - assertEquals("name should not be empty", e.getMessage()); + assertEquals("name", e.getMessage()); } } @@ -1490,7 +1489,7 @@ public void testAddMetadataRecordFileWithEmptyName() throws Exception { backend.addMetadataRecord(tempFile, ""); fail("Expected IllegalArgumentException for empty name"); } catch (IllegalArgumentException e) { - assertEquals("name should not be empty", e.getMessage()); + assertEquals("name", e.getMessage()); } finally { tempFile.delete(); } @@ -1593,9 +1592,9 @@ public void testBlobRequestOptionsConfiguration() { Properties props = getConfigurationWithConnectionString(); // Set various configuration options - props.setProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "8"); - props.setProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT, "45000"); - props.setProperty(AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME, "true"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "8"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_REQUEST_TIMEOUT, "45000"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_NAME, "true"); backend.setProperties(props); @@ -1675,8 +1674,8 @@ public void testDirectAccessMethodsWithEnabledExpiry() throws Exception { Properties props = getConfigurationWithConnectionString(); // Enable expiry for direct access - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); backend.setProperties(props); backend.init(); @@ -1708,8 +1707,8 @@ public void testDirectAccessMethodsWithEnabledExpiry() throws Exception { org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions uploadOptions = org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions.DEFAULT; - // Use a larger file size to ensure we get 2 parts (2 * 256KB = 512KB) - long uploadSize = 2L * 256L * 1024L; // 512KB to ensure 2 parts + // Use a file size larger than 2 * minPartSize (10MB) to ensure we get 2 parts + long uploadSize = 2L * 10L * 1024L * 1024L + 1L; // just over 20MB org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload upload = backend.initiateHttpUpload(uploadSize, 2, uploadOptions); assertNotNull("Upload should not be null when expiry is enabled", upload); @@ -1732,8 +1731,8 @@ public void testDirectAccessWithNullParameters() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); backend.setProperties(props); backend.init(); @@ -1743,7 +1742,7 @@ public void testDirectAccessWithNullParameters() throws Exception { org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions.DEFAULT); fail("Expected NullPointerException for null identifier"); } catch (NullPointerException e) { - assertEquals("identifier must not be null", e.getMessage()); + assertEquals("identifier", e.getMessage()); } // Test createHttpDownloadURI with null options @@ -1752,7 +1751,7 @@ public void testDirectAccessWithNullParameters() throws Exception { new DataIdentifier("test"), null); fail("Expected NullPointerException for null options"); } catch (NullPointerException e) { - assertEquals("downloadOptions must not be null", e.getMessage()); + assertEquals("downloadOptions", e.getMessage()); } // Test initiateHttpUpload with null options @@ -1770,7 +1769,7 @@ public void testUploadValidationEdgeCases() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "1800"); backend.setProperties(props); backend.init(); @@ -1877,7 +1876,7 @@ public void testInitWithConcurrentRequestCountTooLow() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "0"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "0"); backend.setProperties(props); backend.init(); @@ -1891,7 +1890,7 @@ public void testInitWithConcurrentRequestCountTooHigh() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1000"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION, "1000"); backend.setProperties(props); backend.init(); @@ -1906,7 +1905,7 @@ public void testInitWithExistingContainer() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "true"); + props.setProperty(AzureConstantsV8.AZURE_CREATE_CONTAINER, "true"); backend.setProperties(props); backend.init(); @@ -1920,11 +1919,11 @@ public void testInitWithPresignedURISettings() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "1800"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "100"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, "custom-upload.domain.com"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, "custom-download.domain.com"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "1800"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "100"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, "custom-upload.domain.com"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, "custom-download.domain.com"); backend.setProperties(props); backend.init(); @@ -1937,7 +1936,7 @@ public void testInitWithPresignedDownloadURISettingsWithoutCacheSize() throws Ex AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "1800"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "1800"); // Don't set cache max size - should use default (0) backend.setProperties(props); backend.init(); @@ -1951,7 +1950,7 @@ public void testInitWithReferenceKeyCreationDisabled() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.AZURE_REF_ON_INIT, "false"); + props.setProperty(AzureConstantsV8.AZURE_REF_ON_INIT, "false"); backend.setProperties(props); backend.init(); @@ -2218,7 +2217,7 @@ public void testCreateHttpDownloadURIWithCacheDisabled() throws Exception { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); // Don't set cache size - should disable cache backend.setProperties(props); backend.init(); @@ -2253,9 +2252,9 @@ public void testCreateHttpDownloadURIWithVerifyExistsEnabled() throws Exception AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = getConfigurationWithConnectionString(); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "10"); - props.setProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "true"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS, "3600"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "10"); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS, "true"); backend.setProperties(props); backend.init(); @@ -2433,15 +2432,15 @@ public void testInitAzureDSConfigWithAllProperties() throws DataStoreException { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = new Properties(); - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, "test-container"); - props.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString()); - props.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "testaccount"); - props.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "https://testaccount.blob.core.windows.net"); - props.setProperty(AzureConstants.AZURE_SAS, "test-sas-token"); - props.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "test-account-key"); - props.setProperty(AzureConstants.AZURE_TENANT_ID, "test-tenant-id"); - props.setProperty(AzureConstants.AZURE_CLIENT_ID, "test-client-id"); - props.setProperty(AzureConstants.AZURE_CLIENT_SECRET, "test-client-secret"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, "test-container"); + props.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, getConnectionString()); + props.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "testaccount"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "https://testaccount.blob.core.windows.net"); + props.setProperty(AzureConstantsV8.AZURE_SAS, "test-sas-token"); + props.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "test-account-key"); + props.setProperty(AzureConstantsV8.AZURE_TENANT_ID, "test-tenant-id"); + props.setProperty(AzureConstantsV8.AZURE_CLIENT_ID, "test-client-id"); + props.setProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, "test-client-secret"); backend.setProperties(props); @@ -2457,20 +2456,20 @@ public void testInitAzureDSConfigWithAllPropertiesInvalidCredentials() throws Ex AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = new Properties(); - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, "test-container"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, "test-container"); // Use Azurite endpoint but with invalid credentials String invalidConnectionString = UtilsV8.getConnectionString( AzuriteDockerRule.ACCOUNT_NAME, "INVALID_KEY_aW52YWxpZGtleWludmFsaWRrZXlpbnZhbGlka2V5aW52YWxpZGtleQ==", azurite.getBlobEndpoint()); - props.setProperty(AzureConstants.AZURE_CONNECTION_STRING, invalidConnectionString); - props.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); - props.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); - props.setProperty(AzureConstants.AZURE_SAS, "invalid-sas-token"); - props.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "invalid-account-key"); - props.setProperty(AzureConstants.AZURE_TENANT_ID, "invalid-tenant-id"); - props.setProperty(AzureConstants.AZURE_CLIENT_ID, "invalid-client-id"); - props.setProperty(AzureConstants.AZURE_CLIENT_SECRET, "invalid-client-secret"); + props.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, invalidConnectionString); + props.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); + props.setProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); + props.setProperty(AzureConstantsV8.AZURE_SAS, "invalid-sas-token"); + props.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "invalid-account-key"); + props.setProperty(AzureConstantsV8.AZURE_TENANT_ID, "invalid-tenant-id"); + props.setProperty(AzureConstantsV8.AZURE_CLIENT_ID, "invalid-client-id"); + props.setProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, "invalid-client-secret"); backend.setProperties(props); backend.init(); // Should throw DataStoreException due to invalid credentials @@ -2483,18 +2482,18 @@ public void testInitAzureDSConfigWithAllPropertiesInvalidConnectionStringFormat( AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = new Properties(); - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, "test-container"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, "test-container"); // Malformed connection string - missing required fields and invalid format // Still references Azurite endpoint to show it fails before even attempting connection - props.setProperty(AzureConstants.AZURE_CONNECTION_STRING, + props.setProperty(AzureConstantsV8.AZURE_CONNECTION_STRING, "InvalidFormat;BlobEndpoint=" + azurite.getBlobEndpoint() + ";MissingAccountName"); - props.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); - props.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); - props.setProperty(AzureConstants.AZURE_SAS, "test-sas-token"); - props.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "test-account-key"); - props.setProperty(AzureConstants.AZURE_TENANT_ID, "test-tenant-id"); - props.setProperty(AzureConstants.AZURE_CLIENT_ID, "test-client-id"); - props.setProperty(AzureConstants.AZURE_CLIENT_SECRET, "test-client-secret"); + props.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME); + props.setProperty(AzureConstantsV8.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint()); + props.setProperty(AzureConstantsV8.AZURE_SAS, "test-sas-token"); + props.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "test-account-key"); + props.setProperty(AzureConstantsV8.AZURE_TENANT_ID, "test-tenant-id"); + props.setProperty(AzureConstantsV8.AZURE_CLIENT_ID, "test-client-id"); + props.setProperty(AzureConstantsV8.AZURE_CLIENT_SECRET, "test-client-secret"); backend.setProperties(props); backend.init(); // Should throw IllegalArgumentException due to malformed connection string @@ -2506,7 +2505,7 @@ public void testInitAzureDSConfigWithMinimalProperties() { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = new Properties(); - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, "minimal-container"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, "minimal-container"); // Only set container name, all other properties will use empty defaults backend.setProperties(props); @@ -2531,9 +2530,9 @@ public void testInitAzureDSConfigWithPartialProperties() { AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); Properties props = new Properties(); - props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, "partial-container"); - props.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "partialaccount"); - props.setProperty(AzureConstants.AZURE_TENANT_ID, "partial-tenant"); + props.setProperty(AzureConstantsV8.AZURE_BLOB_CONTAINER_NAME, "partial-container"); + props.setProperty(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "partialaccount"); + props.setProperty(AzureConstantsV8.AZURE_TENANT_ID, "partial-tenant"); // Mix of some properties set, others using defaults backend.setProperties(props); @@ -2551,4 +2550,172 @@ public void testInitAzureDSConfigWithPartialProperties() { assertTrue("initAzureDSConfig was called during init", e.getMessage().contains("Invalid connection string")); } } + + // ===================================================================== + // CSO Prevention Tests (Step 3.2) + // + // These tests guard against the root cause of CSO-Release-24893: + // V12 constants (4000 MiB max part) leaking into V8 code path. + // ===================================================================== + + @Test + public void testV8MaxPartSize_MustBe100MiB() { + assertEquals("V8 MAX_MULTIPART_UPLOAD_PART_SIZE must be 100 MiB (CSO root cause guard)", + 100L * 1024 * 1024, + AzureConstantsV8.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE); + } + + @Test + public void testV8MinPartSize_MustBe10MiB() { + assertEquals("V8 MIN_MULTIPART_UPLOAD_PART_SIZE must be 10 MiB", + 10L * 1024 * 1024, + AzureConstantsV8.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE); + } + + @Test + public void testV8MaxBinaryUploadSize_MustBe4_75TiB() { + long expected = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); + assertEquals("V8 MAX_BINARY_UPLOAD_SIZE must be ~4.75 TiB", + expected, + AzureConstantsV8.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE); + } + + @Test + public void testV8BufferedStreamThreshold_MustBe1MiB() { + assertEquals("V8 BUFFERED_STREAM_THRESHOLD must be 1 MiB", + 1024L * 1024, + AzureConstantsV8.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD); + } + + @Test + public void testV8DefaultConcurrentRequestCount_MustBe2() { + assertEquals("V8 DEFAULT_CONCURRENT_REQUEST_COUNT must be 2", + 2, + AzureConstantsV8.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT); + } + + @Test + public void testV8MaxConcurrentRequestCount_MustBe50() { + assertEquals("V8 MAX_CONCURRENT_REQUEST_COUNT must be 50", + 50, + AzureConstantsV8.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT); + } + + @Test + public void testV8InitiateHttpUpload_100MiB_AtMost10URIs() throws Exception { + createBlobContainer(); + + AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); + Properties props = getConfigurationWithConnectionString(); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60"); + backend.setProperties(props); + backend.init(); + + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions uploadOptions = + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions.DEFAULT; + + // 100 MiB with 10 URIs max: ceil(100MiB / 10MiB minPart) = 10, capped at maxNumberOfURIs=10 + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload upload = + backend.initiateHttpUpload(100L * 1024 * 1024, 10, uploadOptions); + + assertNotNull("Upload should not be null", upload); + assertTrue("100 MiB upload with 10 URIs should produce at most 10 URIs", + upload.getUploadURIs().size() <= 10); + assertTrue("100 MiB upload should produce at least 1 URI", + upload.getUploadURIs().size() >= 1); + } + + @Test + public void testV8InitiateHttpUpload_1GiB_MaxPart100MiB() throws Exception { + // DIRECT CSO REPRODUCTION: With V12 constants (4000 MiB max part), + // 1 GiB / 10 URIs = 100 MiB per part which is <= 4000 MiB, so only ~1 URI. + // With V8 constants (100 MiB max part), we get 10 URIs at 100 MiB each. + createBlobContainer(); + + AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); + Properties props = getConfigurationWithConnectionString(); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60"); + backend.setProperties(props); + backend.init(); + + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions uploadOptions = + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions.DEFAULT; + + long oneGiB = 1024L * 1024 * 1024; + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload upload = + backend.initiateHttpUpload(oneGiB, -1, uploadOptions); + + assertNotNull("Upload should not be null", upload); + // With minPartSize=10MiB: ceil(1GiB / 10MiB) = ~103 URIs + // Must NOT be 1 URI (which would happen with V12 min of 256KiB and max of 4000MiB) + int uriCount = upload.getUploadURIs().size(); + assertTrue("1 GiB upload with unlimited URIs must produce multiple URIs (got " + uriCount + ")", + uriCount > 1); + // ceil(1GiB / 10MiB) = 103 + long expectedParts = (long) Math.ceil(((double) oneGiB) / ((double) AzureConstantsV8.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); + assertEquals("1 GiB upload with unlimited URIs should produce ceil(1GiB/10MiB) URIs", + expectedParts, uriCount); + } + + @Test + public void testV8InitiateHttpUpload_4GiB_40URIs() throws Exception { + // 4 GiB with 40 URIs max: requestedPartSize = 4GiB/40 = ~107 MiB > 100 MiB maxPart → exception + // So test with 50 URIs: requestedPartSize = 4GiB/50 = ~86 MiB <= 100 MiB → OK + createBlobContainer(); + + AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); + Properties props = getConfigurationWithConnectionString(); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60"); + backend.setProperties(props); + backend.init(); + + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions uploadOptions = + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions.DEFAULT; + + long fourGiB = 4L * 1024 * 1024 * 1024; + + // With unlimited URIs: ceil(4GiB / 10MiB) = 410 URIs — not 1! + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload upload = + backend.initiateHttpUpload(fourGiB, -1, uploadOptions); + + assertNotNull("Upload should not be null", upload); + int uriCount = upload.getUploadURIs().size(); + long expectedParts = (long) Math.ceil(((double) fourGiB) / ((double) AzureConstantsV8.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); + assertEquals("4 GiB upload with unlimited URIs should produce ceil(4GiB/10MiB) = " + expectedParts + " URIs", + expectedParts, uriCount); + assertTrue("4 GiB upload must produce many URIs, not 1 (got " + uriCount + ")", + uriCount > 10); + } + + @Test + public void testV8InitiateHttpUpload_40GiB_UnlimitedURIs() throws Exception { + // 40 GiB is a realistic DAM archive size that triggered the production OOM. + // With V12's leaked 4000 MiB max part: 40 GiB / 10 MiB min = 4096 URIs, but + // the real danger was the oversized part allocation (4000 MiB buffers). + // With V8's correct 100 MiB max / 10 MiB min: ceil(40 GiB / 10 MiB) = 4096 URIs + // at manageable 10 MiB each — no OOM. + createBlobContainer(); + + AzureBlobStoreBackendV8 backend = new AzureBlobStoreBackendV8(); + Properties props = getConfigurationWithConnectionString(); + props.setProperty(AzureConstantsV8.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60"); + backend.setProperties(props); + backend.init(); + + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions uploadOptions = + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions.DEFAULT; + + long fortyGiB = 40L * 1024 * 1024 * 1024; + org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload upload = + backend.initiateHttpUpload(fortyGiB, -1, uploadOptions); + + assertNotNull("Upload should not be null", upload); + int uriCount = upload.getUploadURIs().size(); + long expectedParts = (long) Math.ceil(((double) fortyGiB) / ((double) AzureConstantsV8.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE)); + assertEquals("40 GiB upload should produce ceil(40GiB/10MiB) = " + expectedParts + " URIs", + expectedParts, uriCount); + assertTrue("40 GiB upload must produce thousands of URIs, not a handful (got " + uriCount + ")", + uriCount > 100); + } + } diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8Test.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8Test.java index 2b2c46f832d..3263a795216 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8Test.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/UtilsV8Test.java @@ -27,7 +27,6 @@ import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.junit.After; import org.junit.Test; import org.mockito.MockedStatic; @@ -76,7 +75,7 @@ public void testPrivateConstructor() throws Exception { @Test public void testConnectionStringIsBasedOnProperty() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_CONNECTION_STRING, "DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey"); + properties.put(AzureConstantsV8.AZURE_CONNECTION_STRING, "DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey"); String connectionString = UtilsV8.getConnectionStringFromProperties(properties); assertEquals("DefaultEndpointsProtocol=https;AccountName=accountName;AccountKey=accountKey", connectionString); } @@ -84,8 +83,8 @@ public void testConnectionStringIsBasedOnProperty() { @Test public void testConnectionStringIsBasedOnSAS() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_SAS, "sas"); - properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint"); + properties.put(AzureConstantsV8.AZURE_SAS, "sas"); + properties.put(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "endpoint"); String connectionString = UtilsV8.getConnectionStringFromProperties(properties); assertEquals(connectionString, String.format("BlobEndpoint=%s;SharedAccessSignature=%s", "endpoint", "sas")); @@ -94,8 +93,8 @@ public void testConnectionStringIsBasedOnSAS() { @Test public void testConnectionStringIsBasedOnSASWithoutEndpoint() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_SAS, "sas"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "account"); + properties.put(AzureConstantsV8.AZURE_SAS, "sas"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "account"); String connectionString = UtilsV8.getConnectionStringFromProperties(properties); assertEquals(connectionString, String.format("AccountName=%s;SharedAccessSignature=%s", "account", "sas")); @@ -104,8 +103,8 @@ public void testConnectionStringIsBasedOnSASWithoutEndpoint() { @Test public void testConnectionStringIsBasedOnAccessKeyIfSASMissing() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); String connectionString = UtilsV8.getConnectionStringFromProperties(properties); assertEquals(connectionString, @@ -115,11 +114,11 @@ public void testConnectionStringIsBasedOnAccessKeyIfSASMissing() { @Test public void testConnectionStringSASIsPriority() { Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_SAS, "sas"); - properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint"); + properties.put(AzureConstantsV8.AZURE_SAS, "sas"); + properties.put(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "endpoint"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "accessKey"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "secretKey"); String connectionString = UtilsV8.getConnectionStringFromProperties(properties); assertEquals(connectionString, @@ -137,8 +136,8 @@ public void testConnectionStringFromPropertiesWithEmptyProperties() { public void testConnectionStringFromPropertiesWithNullValues() { Properties properties = new Properties(); // Properties.put() doesn't accept null values, so we test with empty strings instead - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, ""); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, ""); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, ""); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, ""); String connectionString = UtilsV8.getConnectionStringFromProperties(properties); assertEquals("DefaultEndpointsProtocol=https;AccountName=;AccountKey=", connectionString); } @@ -232,8 +231,8 @@ public void testGetRetryPolicyWithEmptyString() { @Test public void testSetProxyIfNeededWithValidProxySettings() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, "proxy.example.com"); - properties.setProperty(AzureConstants.PROXY_PORT, "8080"); + properties.setProperty(AzureConstantsV8.PROXY_HOST, "proxy.example.com"); + properties.setProperty(AzureConstantsV8.PROXY_PORT, "8080"); UtilsV8.setProxyIfNeeded(properties); @@ -250,7 +249,7 @@ public void testSetProxyIfNeededWithValidProxySettings() { @Test public void testSetProxyIfNeededWithMissingProxyHost() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_PORT, "8080"); + properties.setProperty(AzureConstantsV8.PROXY_PORT, "8080"); UtilsV8.setProxyIfNeeded(properties); assertNull("Proxy should not be set when host is missing", OperationContext.getDefaultProxy()); @@ -259,7 +258,7 @@ public void testSetProxyIfNeededWithMissingProxyHost() { @Test public void testSetProxyIfNeededWithMissingProxyPort() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, "proxy.example.com"); + properties.setProperty(AzureConstantsV8.PROXY_HOST, "proxy.example.com"); // Missing port property - proxy should not be set UtilsV8.setProxyIfNeeded(properties); @@ -276,8 +275,8 @@ public void testSetProxyIfNeededWithEmptyProperties() { @Test public void testSetProxyIfNeededWithNullHost() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, ""); - properties.setProperty(AzureConstants.PROXY_PORT, "8080"); + properties.setProperty(AzureConstantsV8.PROXY_HOST, ""); + properties.setProperty(AzureConstantsV8.PROXY_PORT, "8080"); UtilsV8.setProxyIfNeeded(properties); assertNull("Proxy should not be set with empty host", OperationContext.getDefaultProxy()); @@ -286,8 +285,8 @@ public void testSetProxyIfNeededWithNullHost() { @Test public void testSetProxyIfNeededWithEmptyPort() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, "proxy.example.com"); - properties.setProperty(AzureConstants.PROXY_PORT, ""); + properties.setProperty(AzureConstantsV8.PROXY_HOST, "proxy.example.com"); + properties.setProperty(AzureConstantsV8.PROXY_PORT, ""); // Empty port string - proxy should not be set UtilsV8.setProxyIfNeeded(properties); @@ -297,8 +296,8 @@ public void testSetProxyIfNeededWithEmptyPort() { @Test(expected = NumberFormatException.class) public void testSetProxyIfNeededWithInvalidPort() { Properties properties = new Properties(); - properties.setProperty(AzureConstants.PROXY_HOST, "proxy.example.com"); - properties.setProperty(AzureConstants.PROXY_PORT, "invalid"); + properties.setProperty(AzureConstantsV8.PROXY_HOST, "proxy.example.com"); + properties.setProperty(AzureConstantsV8.PROXY_PORT, "invalid"); // After the bug fix, this should now throw NumberFormatException UtilsV8.setProxyIfNeeded(properties); @@ -491,11 +490,11 @@ public void testGetBlobContainerWithStorageException() throws Exception { public void testConnectionStringPriorityOrder() { // Test that connection string has highest priority Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_CONNECTION_STRING, "connection-string-value"); - properties.put(AzureConstants.AZURE_SAS, "sas-value"); - properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint-value"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "account-value"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "key-value"); + properties.put(AzureConstantsV8.AZURE_CONNECTION_STRING, "connection-string-value"); + properties.put(AzureConstantsV8.AZURE_SAS, "sas-value"); + properties.put(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "endpoint-value"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "account-value"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "key-value"); String result = UtilsV8.getConnectionStringFromProperties(properties); assertEquals("connection-string-value", result); @@ -505,10 +504,10 @@ public void testConnectionStringPriorityOrder() { public void testSASPriorityOverAccountKey() { // Test that SAS has priority over account key Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_SAS, "sas-value"); - properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint-value"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "account-value"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "key-value"); + properties.put(AzureConstantsV8.AZURE_SAS, "sas-value"); + properties.put(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "endpoint-value"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "account-value"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "key-value"); String result = UtilsV8.getConnectionStringFromProperties(properties); assertEquals("BlobEndpoint=endpoint-value;SharedAccessSignature=sas-value", result); @@ -518,9 +517,9 @@ public void testSASPriorityOverAccountKey() { public void testFallbackToAccountKey() { // Test fallback to account key when no connection string or SAS Properties properties = new Properties(); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, "account-value"); - properties.put(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, "key-value"); - properties.put(AzureConstants.AZURE_BLOB_ENDPOINT, "endpoint-value"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_NAME, "account-value"); + properties.put(AzureConstantsV8.AZURE_STORAGE_ACCOUNT_KEY, "key-value"); + properties.put(AzureConstantsV8.AZURE_BLOB_ENDPOINT, "endpoint-value"); String result = UtilsV8.getConnectionStringFromProperties(properties); assertEquals("DefaultEndpointsProtocol=https;AccountName=account-value;AccountKey=key-value;BlobEndpoint=endpoint-value", result); diff --git a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg index f8181f92fb2..9de8f50d9c2 100644 --- a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg +++ b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.jcr.osgi.RepositoryManager.cfg @@ -1,15 +1,15 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Empty config to trigger default setup +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#Empty config to trigger default setup diff --git a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg index ccc6a4fde9b..d05269419a0 100644 --- a/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg +++ b/oak-it-osgi/src/test/config/org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.cfg @@ -1,16 +1,16 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name=Oak -repository.home=target/tarmk +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name=Oak +repository.home=target/tarmk diff --git a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java index 3d806c46f80..6dd6fe0659c 100644 --- a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java +++ b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/AzureDataStoreFixture.java @@ -26,10 +26,9 @@ import org.apache.jackrabbit.oak.spi.blob.data.DataStore; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainer; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainers; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.UtilsV8; -import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier; import org.apache.jackrabbit.oak.fixture.NodeStoreFixture; import org.apache.jackrabbit.oak.jcr.binary.fixtures.nodestore.FixtureUtils; import org.jetbrains.annotations.NotNull; @@ -37,9 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.azure.storage.blob.BlobContainerClient; /** * Fixture for AzureDataStore based on an azure.properties config file. It creates @@ -67,8 +63,7 @@ public class AzureDataStoreFixture implements DataStoreFixture { @Nullable private final Properties azProps; - private final Map containers = new HashMap<>(); - private static final String AZURE_SDK_12_ENABLED = "blob.azure.v12.enabled"; + private final Map containers = new HashMap<>(); public AzureDataStoreFixture() { azProps = FixtureUtils.loadDataStoreProperties("azure.config", "azure.properties", ".azure"); @@ -96,26 +91,11 @@ public DataStore createDataStore() { log.info("Creating Azure test blob container {}", containerName); - String connectionString = Utils.getConnectionStringFromProperties(azProps); try { - boolean useSDK12 = SystemPropertySupplier.create(AZURE_SDK_12_ENABLED, false).get(); - Object container; - - if (useSDK12) { - log.info("Starting blob store using azure sdk 12"); - BlobContainerClient containerClient = Utils.getBlobContainer(connectionString, containerName, null, azProps); - containerClient.createIfNotExists(); - container = containerClient; - } else { - log.info("Starting blob store using azure sdk 8"); - CloudBlobContainer blobContainer = UtilsV8.getBlobContainer(connectionString, containerName); - blobContainer.createIfNotExists(); - container = blobContainer; - } - - // create new properties since azProps is shared for all created DataStores Properties clonedAzProps = new Properties(azProps); clonedAzProps.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, containerName); + AzureBlobContainer container = AzureBlobContainers.create(clonedAzProps); + container.createIfNotExists(); // setup Oak DS AzureDataStore dataStore = new AzureDataStore(); @@ -125,7 +105,7 @@ public DataStore createDataStore() { containers.put(dataStore, container); return dataStore; - } catch (DataStoreException | StorageException e) { + } catch (Exception e) { throw new AssertionError("Azure DataStore fixture fails because of issue with Azure config or connection", e); } } @@ -142,18 +122,11 @@ public void dispose(DataStore dataStore) { log.warn("Issue while disposing DataStore", e); } - Object container = containers.get(dataStore); + AzureBlobContainer container = containers.get(dataStore); if (container != null) { - try { - if (container instanceof CloudBlobContainer) { - CloudBlobContainer blobContainer = (CloudBlobContainer) container; - log.info("Removing Azure test blob container {}", blobContainer.getName()); - blobContainer.delete(); - } else if (container instanceof BlobContainerClient) { - BlobContainerClient containerClient = (BlobContainerClient) container; - log.info("Removing Azure test blob container {}", containerClient.getBlobContainerName()); - containerClient.delete(); - } + try (AzureBlobContainer closableContainer = container) { + log.info("Removing Azure test blob container {}", container.getName()); + closableContainer.delete(); } catch (Exception e) { log.warn("Unable to delete Azure Blob container", e); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java index de5cba57d43..170eaf635de 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java @@ -18,12 +18,11 @@ */ package org.apache.jackrabbit.oak.fixture; -import com.azure.storage.blob.BlobContainerClient; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataStore; -import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainerProvider; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainer; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainers; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore; import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants; @@ -124,22 +123,25 @@ public static void deleteAzureContainer(Map config, String containerN log.warn("container name is null or blank, cannot initialize blob container"); return; } - BlobContainerClient container = getBlobContainerClient(config, containerName); - if (container == null) { + Properties properties = toAzureProperties(config, containerName); + if (properties == null) { log.warn("cannot delete the container as it is not initialized"); return; } - log.info("deleting container [{}]", containerName); - if (container.deleteIfExists()) { - log.info("container [{}] deleted", containerName); - } else { - log.info("container [{}] doesn't exists", containerName); + AzureBlobContainer container = AzureBlobContainers.getReference(properties); + try (AzureBlobContainer closableContainer = container) { + log.info("deleting container [{}]", containerName); + if (closableContainer.deleteIfExists()) { + log.info("container [{}] deleted", containerName); + } else { + log.info("container [{}] doesn't exists", containerName); + } } } @Nullable - private static BlobContainerClient getBlobContainerClient(@NotNull Map config, - @NotNull String containerName) throws DataStoreException { + private static Properties toAzureProperties(@NotNull Map config, + @NotNull String containerName) { final String azureConnectionString = (String) config.get(AzureConstants.AZURE_CONNECTION_STRING); final String clientId = (String) config.get(AzureConstants.AZURE_CLIENT_ID); final String clientSecret = (String) config.get(AzureConstants.AZURE_CLIENT_SECRET); @@ -154,17 +156,37 @@ private static BlobContainerClient getBlobContainerClient(@NotNull Map { + if (k != null && v != null) { + properties.put(k, v); + } + }); + properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, containerName); + if (azureConnectionString != null) { + properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, azureConnectionString); + } + if (accountName != null) { + properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, accountName); + } + if (accountKey != null) { + properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, accountKey); + } + if (blobEndpoint != null) { + properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, blobEndpoint); + } + if (sasToken != null) { + properties.setProperty(AzureConstants.AZURE_SAS, sasToken); + } + if (clientId != null) { + properties.setProperty(AzureConstants.AZURE_CLIENT_ID, clientId); + } + if (clientSecret != null) { + properties.setProperty(AzureConstants.AZURE_CLIENT_SECRET, clientSecret); + } + if (tenantId != null) { + properties.setProperty(AzureConstants.AZURE_TENANT_ID, tenantId); + } + return properties; } -} \ No newline at end of file +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsIT.java similarity index 91% rename from oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsTest.java rename to oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsIT.java index 95fcdc856d5..ad120286cc6 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/fixture/DataStoreUtilsIT.java @@ -23,9 +23,9 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.Appender; import ch.qos.logback.core.read.ListAppender; -import com.azure.storage.blob.BlobContainerClient; import org.apache.commons.lang3.StringUtils; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainerProvider; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainer; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainers; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import org.jetbrains.annotations.NotNull; @@ -36,8 +36,6 @@ import org.junit.Test; import org.slf4j.LoggerFactory; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -45,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Properties; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -53,7 +52,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -public class DataStoreUtilsTest { +public class DataStoreUtilsIT { @ClassRule public static AzuriteDockerRule azuriteDockerRule = new AzuriteDockerRule(); @@ -71,18 +70,20 @@ public class DataStoreUtilsTest { private static final String CONTAINER_DELETED_MESSAGE = "container [%s] deleted"; private static final String DELETING_CONTAINER_MESSAGE = "deleting container [%s]"; - private BlobContainerClient container; + private AzureBlobContainer container; @Before - public void init() throws URISyntaxException, InvalidKeyException { - container = azuriteDockerRule.getContainer(CONTAINER_NAME, String.format(AZURE_CONNECTION_STRING, AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azuriteDockerRule.getBlobEndpoint())); + public void init() throws Exception { + container = azuriteDockerRule.getAzureBlobContainer(CONTAINER_NAME); assertTrue(container.exists()); } @After - public void cleanup() { + public void cleanup() throws Exception { if (container != null) { - container.deleteIfExists(); + try (AzureBlobContainer closableContainer = container) { + closableContainer.deleteIfExists(); + } } } @@ -154,17 +155,12 @@ public void delete_container_service_principal() throws Exception { Assume.assumeNotNull(clientSecret); Assume.assumeNotNull(tenantId); - BlobContainerClient container; - AzureBlobContainerProvider azureBlobContainerProvider = AzureBlobContainerProvider.Builder.builder(CONTAINER_NAME) - .withAccountName(accountName) - .withClientId(clientId) - .withClientSecret(clientSecret) - .withTenantId(tenantId).build(); - container = azureBlobContainerProvider.getBlobContainer(); - container.createIfNotExists(); + try (AzureBlobContainer container = AzureBlobContainers.create(toProperties(getConfigMap(null, accountName, null, null, null, clientId, clientSecret, tenantId), CONTAINER_NAME))) { + container.createIfNotExists(); - assertNotNull(container); - assertTrue(container.exists()); + assertNotNull(container); + assertTrue(container.exists()); + } ListAppender logAppender = subscribeAppender(); @@ -239,6 +235,15 @@ private String getEnvironmentVariable(String variableName) { return System.getenv(variableName); } + private Properties toProperties(Map config, String containerName) { + Properties properties = new Properties(); + config.entrySet().stream() + .filter(entry -> entry.getValue() != null) + .forEach(entry -> properties.put(entry.getKey(), entry.getValue())); + properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, containerName); + return properties; + } + private Map getConfigMap(String connectionString, String accountName, String accessKey, @@ -275,4 +280,4 @@ private void unsubscribe(@NotNull final Appender appender) { ((LoggerContext) LoggerFactory.getILoggerFactory()).getLogger( ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME).detachAppender(appender); } -} \ No newline at end of file +} diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandIT.java similarity index 79% rename from oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java rename to oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandIT.java index b96e6c4bd5e..d53b853789d 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandIT.java @@ -16,13 +16,10 @@ */ package org.apache.jackrabbit.oak.run; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions; -import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainer; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; -import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; @@ -36,20 +33,16 @@ import java.nio.file.Path; import java.time.Duration; import java.time.Instant; -import java.util.Date; -import java.util.EnumSet; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST; -import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -public class DataStoreCopyCommandTest { +public class DataStoreCopyCommandIT { @ClassRule public static AzuriteDockerRule AZURITE = new AzuriteDockerRule(); @@ -67,7 +60,7 @@ public class DataStoreCopyCommandTest { @Rule public TemporaryFolder outDir = new TemporaryFolder(new File("target")); - private CloudBlobContainer container; + private AzureBlobContainer container; @Before public void setUp() throws Exception { @@ -87,7 +80,7 @@ public void missingRequiredOptions() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); cmd.execute( "--source-repo", - container.getUri().toURL().toString() + java.net.URI.create(container.getContainerUri()).toURL().toString() ); } @@ -96,7 +89,7 @@ public void unauthenticated() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--include-path", BLOB1, "--out-dir", @@ -109,11 +102,11 @@ public void singleBlobWithIncludePath() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--include-path", BLOB1, "--sas-token", - container.generateSharedAccessSignature(policy(EnumSet.of(READ, LIST)), null), + container.generateSharedAccessSignature(Instant.now().plus(Duration.ofDays(7))), "--out-dir", outDir.getRoot().getAbsolutePath() ); @@ -139,11 +132,11 @@ public void allBlobsWithFileIncludePath() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--file-include-path", blobs.toString(), "--sas-token", - container.generateSharedAccessSignature(policy(EnumSet.of(READ, LIST)), null), + container.generateSharedAccessSignature(Instant.now().plus(Duration.ofDays(7))), "--out-dir", outDir.getRoot().getAbsolutePath() ); @@ -161,11 +154,11 @@ public void allBlobsPlusMissingOne() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--file-include-path", blobs.toString(), "--sas-token", - container.generateSharedAccessSignature(policy(EnumSet.of(READ, LIST)), null), + container.generateSharedAccessSignature(Instant.now().plus(Duration.ofDays(7))), "--out-dir", outDir.getRoot().getAbsolutePath() ); @@ -182,11 +175,11 @@ public void onlyFailures() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); assertThrows(RuntimeException.class, () -> cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--file-include-path", blobs.toString(), "--sas-token", - container.generateSharedAccessSignature(policy(EnumSet.of(READ, LIST)), null), + container.generateSharedAccessSignature(Instant.now().plus(Duration.ofDays(7))), "--out-dir", outDir.getRoot().getAbsolutePath() )); @@ -200,11 +193,11 @@ public void allBlobsPlusMissingOneWithFailOnError() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); assertThrows(RuntimeException.class, () -> cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--file-include-path", blobs.toString(), "--sas-token", - container.generateSharedAccessSignature(policy(EnumSet.of(READ, LIST)), null), + container.generateSharedAccessSignature(Instant.now().plus(Duration.ofDays(7))), "--out-dir", outDir.getRoot().getAbsolutePath(), "--fail-on-error", @@ -217,7 +210,7 @@ public void destinationFromBlobId() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); cmd.parseCommandLineParams( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--include-path", BLOB1, "--out-dir", @@ -237,11 +230,11 @@ public void allBlobsWithBogusChecksumAlgorithm() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); assertThrows(RuntimeException.class, () -> cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--file-include-path", blobs.toString(), "--sas-token", - container.generateSharedAccessSignature(policy(EnumSet.of(READ, LIST)), null), + container.generateSharedAccessSignature(Instant.now().plus(Duration.ofDays(7))), "--out-dir", outDir.getRoot().getAbsolutePath(), "--checksum", @@ -257,11 +250,11 @@ public void allBlobsWithChecksum() throws Exception { DataStoreCopyCommand cmd = new DataStoreCopyCommand(); cmd.execute( "--source-repo", - container.getUri().toURL().toString(), + java.net.URI.create(container.getContainerUri()).toURL().toString(), "--file-include-path", blobs.toString(), "--sas-token", - container.generateSharedAccessSignature(policy(EnumSet.of(READ, LIST)), null), + container.generateSharedAccessSignature(Instant.now().plus(Duration.ofDays(7))), "--out-dir", outDir.getRoot().getAbsolutePath(), "--checksum", @@ -281,24 +274,11 @@ public void allBlobsWithChecksum() throws Exception { IOUtils.toString(Path.of(cmd.getDestinationFromId(BLOB2)).toUri(), StandardCharsets.UTF_8)); } - private CloudBlobContainer createBlobContainer() throws Exception { - container = AZURITE.getContainer("blobstore"); + private AzureBlobContainer createBlobContainer() throws Exception { + container = AZURITE.getAzureBlobContainer("blobstore"); for (Map.Entry blob : BLOBS_WITH_CONTENT.entrySet()) { - container.getBlockBlobReference(blob.getKey()).uploadText(blob.getValue()); + container.uploadBlockBlob(blob.getKey(), new java.io.ByteArrayInputStream(blob.getValue().getBytes(StandardCharsets.UTF_8)), blob.getValue().getBytes(StandardCharsets.UTF_8).length); } return container; } - - @NotNull - private static SharedAccessBlobPolicy policy(EnumSet permissions, Instant expirationTime) { - SharedAccessBlobPolicy sharedAccessBlobPolicy = new SharedAccessBlobPolicy(); - sharedAccessBlobPolicy.setPermissions(permissions); - sharedAccessBlobPolicy.setSharedAccessExpiryTime(Date.from(expirationTime)); - return sharedAccessBlobPolicy; - } - - @NotNull - private static SharedAccessBlobPolicy policy(EnumSet permissions) { - return policy(permissions, Instant.now().plus(Duration.ofDays(7))); - } } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreIT.java index d2fde3e5938..df85db0ba77 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreIT.java @@ -238,6 +238,7 @@ private void doUnmergedCommitOnRoot(boolean prevNoPropCacheEnabled) throws Excep FailingDocumentStore fs1 = new FailingDocumentStore(ds); PausableDocumentStore store1 = new PausableDocumentStore(fs1); DocumentNodeStore ns1 = builderProvider.newBuilder().setClusterId(1).setAsyncDelay(0).clock(clock) + .setLeaseCheckMode(LeaseCheckMode.DISABLED) .setPrevNoPropCacheFeature(createFeature(prevNoPropCacheEnabled)) .setDocumentStore(store1).build();