From 476e07ef03b518e3c2fe0fe25e8c2e36111d4bbe Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Wed, 15 Jan 2025 15:12:03 -0800 Subject: [PATCH 1/6] clean up extensions; better re-use --- .../tools/mapreduce/DatastoreExtension.java | 97 --------------- .../tools/mapreduce/EndToEndTest.java | 1 - .../tools/mapreduce/EndToEndTestCase.java | 27 +--- .../mapreduce/MapReduceSpecificationTest.java | 2 +- .../tools/mapreduce/MapSettingsTest.java | 2 +- .../tools/mapreduce/MapSpecificationTest.java | 2 +- .../impl/handlers/MapReduceServletTest.java | 2 +- .../impl/shardedjob/LockingTest.java | 2 +- .../impl/util/SerializationUtilTest.java | 4 +- ...gleCloudStorageLevelDbInputReaderTest.java | 3 +- .../GoogleCloudStorageFileOutputTest.java | 4 - .../servlets/ShufflerServletTest.java | 2 +- .../appengine/tools/pipeline/BarrierTest.java | 2 +- .../tools/pipeline/ClientUseTest.java | 3 +- .../tools/pipeline/FanoutTaskTest.java | 1 + .../tools/pipeline/PipelineTest.java | 2 +- .../appengine/tools/pipeline/RetryTest.java | 2 +- .../tools/pipeline/UserGuideTest.java | 2 +- .../impl/backend/AppEngineBackEndTest.java | 2 +- .../impl/backend/AppEngineTaskQueueTest.java | 2 +- .../tools/test/CloudStorageExtensions.java | 115 ++++++++++++++++++ .../tools/test/CloudTasksExtension.java | 60 +++++++++ .../DatastoreExtension.java | 3 +- .../PipelineSetupExtensions.java | 4 +- 24 files changed, 197 insertions(+), 149 deletions(-) delete mode 100644 java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java create mode 100644 java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java create mode 100644 java/src/test/java/com/google/appengine/tools/test/CloudTasksExtension.java rename java/src/test/java/com/google/appengine/tools/{pipeline => test}/DatastoreExtension.java (97%) rename java/src/test/java/com/google/appengine/tools/{mapreduce => test}/PipelineSetupExtensions.java (98%) diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java b/java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java deleted file mode 100644 index 33f9b695..00000000 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.google.appengine.tools.mapreduce; - -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.testing.LocalDatastoreHelper; -import lombok.extern.java.Log; -import org.junit.jupiter.api.extension.*; - -import java.net.ConnectException; -import java.time.Duration; -import java.util.logging.Level; - -/** - * Junit5 extension to initialize local datastore emulator for tests - * Use it in your tests with {@code @ExtendWith(DatastoreExtension.class)} - * - * TODO: replace with setup for all the pipelines stuff?? - */ -@Log -public class DatastoreExtension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback { - - public static String TEST_DATASTORE_PROJECT_ID = "test-project"; - public static String DS_CONTEXT_KEY = "ds-emulator"; - public static String DS_OPTIONS_CONTEXT_KEY = "ds-options"; - - private LocalDatastoreHelper globalDatastoreHelper; - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - globalDatastoreHelper = LocalDatastoreHelper.newBuilder() - .setStoreOnDisk(false) // can't reset if storing data disk - .setConsistency(1.0) - .build(); - globalDatastoreHelper.start(); - log.info("Datastore emulator started on port : " + globalDatastoreHelper.getPort()); - } - - @Override - public void afterAll(ExtensionContext extensionContext) throws Exception { - - int attempt = 0; - boolean stopped = false; - while (!stopped && attempt < 3) { - ++attempt; - try { - org.threeten.bp.Duration timeout = - org.threeten.bp.Duration.ofSeconds(5).multipliedBy(attempt); - globalDatastoreHelper.stop(timeout); - stopped = true; - } catch (ConnectException e) { - // don't want to kill the test, but also don't want to leave the emulator running - log.warning("DatastoreExtension : Failed to connect in order to stop datastore emulator; retrying..."); - } catch (Exception e) { - log.log(Level.WARNING, "DatastoreExtension : Failed to stop datastore emulator; retrying...", e); - } - } - log.info("Datastore emulator stopped"); - } - - @Override - public void beforeEach(ExtensionContext extensionContext) throws Exception { - globalDatastoreHelper.reset(); - log.info("Datastore emulator reset"); - DatastoreOptions options = globalDatastoreHelper.getOptions().toBuilder() - .setProjectId(TEST_DATASTORE_PROJECT_ID) - .build(); - - extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).put(DS_OPTIONS_CONTEXT_KEY, options); - - Datastore datastore = options.getService(); - extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).put(DS_CONTEXT_KEY, datastore); - } - - public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { - - @Override - public boolean supportsParameter(ParameterContext parameterContext, - ExtensionContext extensionContext) throws ParameterResolutionException { - return parameterContext.getParameter().getType() == Datastore.class - || parameterContext.getParameter().getType() == DatastoreOptions.class; - } - - @Override - public Object resolveParameter(ParameterContext parameterContext, - ExtensionContext extensionContext) throws ParameterResolutionException { - if (parameterContext.getParameter().getType() == Datastore.class) { - return extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).get(DatastoreExtension.DS_CONTEXT_KEY); - } else if (parameterContext.getParameter().getType() == DatastoreOptions.class) { - return extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).get(DatastoreExtension.DS_OPTIONS_CONTEXT_KEY); - } else { - throw new ParameterResolutionException("Unsupported parameter type: " + parameterContext.getParameter().getType()); - } - } - } - -} - diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java index 3f6d5203..802b21b0 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java @@ -75,7 +75,6 @@ public class EndToEndTest extends EndToEndTestCase { private static final Logger log = Logger.getLogger(EndToEndTest.class.getName()); - GoogleCloudStorageFileOutput.Options cloudStorageFileOutputOptions; MapReduceSettings testSettings; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java index 933bd6c4..f3249eff 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java @@ -10,9 +10,6 @@ import com.google.appengine.api.taskqueue.dev.QueueStateInfo; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.HeaderWrapper; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; -import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; -import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.impl.shardedjob.IncrementalTaskId; import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobHandler; import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunId; @@ -22,12 +19,12 @@ import com.google.appengine.tools.pipeline.TestUtils; import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; import com.google.appengine.tools.pipeline.impl.servlets.TaskHandler; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.cloud.datastore.Datastore; import com.google.common.base.CharMatcher; import lombok.Getter; import lombok.Setter; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import java.io.ByteArrayInputStream; @@ -49,19 +46,9 @@ public abstract class EndToEndTestCase { private static final Logger logger = Logger.getLogger(EndToEndTestCase.class.getName()); - private final LocalServiceTestHelper helper = - new LocalServiceTestHelper( - new LocalTaskQueueTestConfig().setDisableAutoTaskExecution(true), - // don't think we use memcache - //new LocalMemcacheServiceTestConfig(), - new LocalModulesServiceTestConfig()); + @Setter(onMethod_ = @BeforeEach) private LocalTaskQueue taskQueue; - /** Implement in sub-classes to set system environment properties for tests. */ - protected Map getEnvAttributes() throws Exception { - return null; - } - @Getter @Setter(onMethod_ = @BeforeEach) Datastore datastore; @@ -83,12 +70,6 @@ protected Map getEnvAttributes() throws Exception { @BeforeEach public void setUp() throws Exception { - helper.setUp(); - Map envAttributes = getEnvAttributes(); - if (envAttributes != null) { - LocalServiceTestHelper.getApiProxyLocal().appendProperties(envAttributes); - } - taskQueue = LocalTaskQueueTestConfig.getLocalTaskQueue(); // Creating files is not allowed in some test execution environments, so don't. storageTestHelper = new CloudStorageIntegrationTestHelper(); storageTestHelper.setUp(); @@ -97,10 +78,6 @@ public void setUp() throws Exception { mrServlet.init(); } - @AfterEach - public void tearDown() throws Exception { - helper.tearDown(); - } public ShardedJobRunId shardedJobId(String jobId) { return ShardedJobRunId.of( diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java index e6ef60c9..990737d9 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java @@ -5,12 +5,12 @@ import com.google.appengine.tools.mapreduce.impl.InProcessMapReduce; import com.google.appengine.tools.pipeline.PipelineService; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.common.collect.ImmutableList; import lombok.Getter; import lombok.Setter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import java.util.ArrayList; import java.util.Arrays; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java index 20e3c5f0..a3529ecf 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java @@ -22,11 +22,11 @@ import com.google.appengine.tools.pipeline.JobSetting.OnService; import com.google.appengine.tools.pipeline.JobSetting.OnQueue; import com.google.appengine.tools.pipeline.JobSetting.StatusConsoleUrl; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.apphosting.api.ApiProxy; import com.google.apphosting.api.ApiProxy.Environment; import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.Key; import org.easymock.EasyMock; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java index 61a8fb03..4592077e 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java @@ -2,12 +2,12 @@ import com.google.appengine.tools.mapreduce.impl.InProcessMap; import com.google.appengine.tools.pipeline.PipelineService; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.common.collect.ImmutableList; import lombok.Getter; import lombok.Setter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import java.util.ArrayList; import java.util.Arrays; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java index 50815fda..2d7c3e90 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java @@ -29,7 +29,7 @@ import com.google.appengine.tools.mapreduce.MapReduceJob; import com.google.appengine.tools.mapreduce.MapReduceServlet; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.TestUtils; import com.google.appengine.tools.pipeline.TestingTaskQueueCallback; import com.google.appengine.tools.pipeline.impl.PipelineManager; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java index 5a1195ea..c60d42d8 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java @@ -6,7 +6,7 @@ import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; import com.google.appengine.tools.mapreduce.EndToEndTestCase; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.impl.backend.AppEngineBackEnd; import com.google.apphosting.api.ApiProxy; import com.google.apphosting.api.ApiProxy.Environment; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java index ab3c6c39..2463398a 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java @@ -7,7 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.mapreduce.DatastoreExtension; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.Entity; @@ -23,7 +23,6 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.List; import java.util.Random; /** @@ -31,7 +30,6 @@ */ @ExtendWith({ DatastoreExtension.class, - //AppEngineEnvironmentExtension.class, DatastoreExtension.ParameterResolver.class, }) public class SerializationUtilTest { diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java index 754b84e1..da0439e2 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java @@ -1,11 +1,10 @@ package com.google.appengine.tools.mapreduce.inputs; -import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; import com.google.appengine.tools.mapreduce.GcsFilename; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.mapreduce.impl.util.LevelDbConstants; import com.google.appengine.tools.mapreduce.impl.util.SerializationUtil; import com.google.appengine.tools.mapreduce.outputs.GoogleCloudStorageFileOutput; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java index 9e127940..adf5ad8e 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java @@ -40,10 +40,6 @@ public class GoogleCloudStorageFileOutputTest { @Getter CloudStorageIntegrationTestHelper storageIntegrationTestHelper; - @BeforeClass - public static void setupStorage() { - - } @BeforeEach protected void setUp() throws Exception { storageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java index 466e9674..96f38d35 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java @@ -27,7 +27,6 @@ import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.*; -import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunId; import com.google.appengine.tools.mapreduce.impl.sort.LexicographicalComparator; import com.google.appengine.tools.mapreduce.impl.util.RequestUtils; import com.google.appengine.tools.mapreduce.inputs.GoogleCloudStorageLevelDbInputReader; @@ -40,6 +39,7 @@ import com.google.appengine.tools.pipeline.PipelineService; import com.google.appengine.tools.pipeline.di.JobRunServiceComponent; import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.apphosting.api.ApiProxy; import com.google.cloud.ReadChannel; import com.google.cloud.datastore.Datastore; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java index 9eadb013..e4807d0d 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java @@ -17,6 +17,7 @@ import static com.google.appengine.tools.pipeline.impl.util.GUIDGenerator.USE_SIMPLE_GUIDS_FOR_DEBUGGING; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.cloud.datastore.Key; import com.google.appengine.tools.pipeline.impl.model.Barrier; import com.google.appengine.tools.pipeline.impl.model.Slot; @@ -26,7 +27,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.AfterEach; import java.util.ArrayList; import java.util.List; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java index 7309b084..3569c14b 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java @@ -1,10 +1,9 @@ package com.google.appengine.tools.pipeline; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.impl.PipelineManager; import com.google.appengine.tools.pipeline.impl.backend.AppEngineBackEnd; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; @PipelineSetupExtensions public class ClientUseTest { diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java index 489c49ba..3d67538b 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java @@ -16,6 +16,7 @@ import static com.google.appengine.tools.pipeline.impl.util.GUIDGenerator.USE_SIMPLE_GUIDS_FOR_DEBUGGING; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java index dec5f496..792d56ef 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java @@ -23,7 +23,7 @@ import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.di.JobRunServiceComponent; import com.google.appengine.tools.pipeline.impl.PipelineManager; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java index a524a667..63570b5b 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java @@ -20,7 +20,7 @@ import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.JobSetting.BackoffFactor; import com.google.appengine.tools.pipeline.JobSetting.BackoffSeconds; import com.google.appengine.tools.pipeline.JobSetting.MaxAttempts; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java index 783eb16e..5ce325e5 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java @@ -20,7 +20,7 @@ import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.demo.UserGuideExamples.ComplexJob; import com.google.appengine.tools.pipeline.impl.PipelineManager; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java index 1529740f..0aa921ea 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java @@ -1,6 +1,6 @@ package com.google.appengine.tools.pipeline.impl.backend; -import com.google.appengine.tools.pipeline.DatastoreExtension; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.appengine.tools.pipeline.impl.model.Slot; import com.google.cloud.datastore.Blob; import com.google.cloud.datastore.Datastore; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java index b0ecda03..40dc1a2a 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java @@ -1,6 +1,6 @@ package com.google.appengine.tools.pipeline.impl.backend; -import com.google.appengine.tools.pipeline.DatastoreExtension; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.cloud.datastore.Key; import com.google.appengine.api.taskqueue.TaskHandle; import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java new file mode 100644 index 00000000..20ecb377 --- /dev/null +++ b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java @@ -0,0 +1,115 @@ +package com.google.appengine.tools.test; + +import com.google.auth.Credentials; +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.testing.RemoteStorageHelper; +import org.junit.jupiter.api.extension.*; + +import java.io.ByteArrayInputStream; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.Parameter; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Copyright 2025 Worklytics, Co. + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith({ + CloudStorageExtension.class, + CloudStorageExtension.ParameterResolver.class, +}) +public @interface CloudStorageExtensions { + +} + +class CloudStorageExtension implements BeforeAllCallback { + + public static final String KEY_ENV_VAR = "CI_SERVICE_ACCOUNT_KEY"; + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + String keyVar = System.getenv(KEY_ENV_VAR); + Credentials credentials; + String projectId; + if (keyVar == null) { + //attempt w default credentials + credentials = StorageOptions.getDefaultInstance().getCredentials(); + + //TODO: more elegant solution? weirdness seems to happen if mix projects; credentials' project + // isn't exposed to java code via any public interface; yet bucket is created in the project + // to which the credentials default project is set + projectId = "worklytics-ci"; + //throw new IllegalStateException("Must set environment variable " + KEY_ENV_VAR + " as base64 encoded service account key to use for storage integration tests"); + } else { + String base64EncodedServiceAccountKey = keyVar.trim(); + String jsonKey = new String(Base64.getDecoder().decode(base64EncodedServiceAccountKey.getBytes())); + + credentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream(jsonKey.getBytes())); + projectId = ((ServiceAccountCredentials) credentials).getProjectId(); + } + + Storage storage = StorageOptions.newBuilder() + .setCredentials(credentials) + .setProjectId(projectId) + .build().getService(); + + context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put(Storage.class, storage); + + String bucket = context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get("bucket", String.class); + if (bucket == null) { + bucket = RemoteStorageHelper.generateBucketName(); + context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put("bucket", bucket); + + //avoid test data being retained forever, even if bucket deletion post-test fails + storage.create(BucketInfo.newBuilder(bucket) + .setLifecycleRules(defaultTestDataLifecycle()) + .setSoftDeletePolicy(null) // no soft-deletion + .build()); + + //delete bucket at shutdown + final String bucketToDelete = bucket; + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + RemoteStorageHelper.forceDelete(storage, bucketToDelete, 5, TimeUnit.SECONDS); + } catch (Throwable e) { + Logger.getAnonymousLogger().log(Level.WARNING, "Failed to cleanup bucket: " + bucketToDelete); + } + })); + } + } + + static List defaultTestDataLifecycle() { + return Collections.singletonList(new BucketInfo.LifecycleRule( + BucketInfo.LifecycleRule.LifecycleAction.newDeleteAction(), + BucketInfo.LifecycleRule.LifecycleCondition.newBuilder().setAge(30).build())); + } + + public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + return extensionContext.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get(Storage.class); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + Parameter parameter = parameterContext.getParameter(); + return Storage.class.equals(parameter.getType()); + } + } + +} diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudTasksExtension.java b/java/src/test/java/com/google/appengine/tools/test/CloudTasksExtension.java new file mode 100644 index 00000000..713d3cb9 --- /dev/null +++ b/java/src/test/java/com/google/appengine/tools/test/CloudTasksExtension.java @@ -0,0 +1,60 @@ +package com.google.appengine.tools.test; + +import com.google.appengine.api.taskqueue.dev.LocalTaskQueue; +import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; +import com.google.appengine.tools.development.testing.LocalServiceTestHelper; +import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; +import org.junit.jupiter.api.extension.*; + +import java.lang.reflect.Parameter; + +/** + * Copyright 2025 Worklytics, Co. + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +public class CloudTasksExtension implements BeforeAllCallback, BeforeEachCallback, AfterEachCallback { + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + LocalServiceTestHelper helper = + new LocalServiceTestHelper( + new LocalTaskQueueTestConfig().setDisableAutoTaskExecution(true), + new LocalModulesServiceTestConfig() //yeah, this is still here ... + ); + context.getStore(ExtensionContext.Namespace.create(getClass())).put(LocalServiceTestHelper.class, helper); + } + + @Override + public void beforeEach(ExtensionContext context) throws Exception { + LocalServiceTestHelper helper = + (LocalServiceTestHelper) context.getStore(ExtensionContext.Namespace.create(getClass())).get(LocalServiceTestHelper.class); + + helper.setUp(); + + LocalTaskQueue taskQueue = LocalTaskQueueTestConfig.getLocalTaskQueue(); + + context.getStore(ExtensionContext.Namespace.create(getClass())).put(LocalTaskQueue.class, taskQueue); + } + + @Override + public void afterEach(ExtensionContext context) throws Exception { + LocalServiceTestHelper helper = + (LocalServiceTestHelper) context.getStore(ExtensionContext.Namespace.create(getClass())).get(LocalServiceTestHelper.class); + + helper.tearDown(); + } + + public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + return extensionContext.getStore(ExtensionContext.Namespace.create(CloudTasksExtension.class)).get(LocalTaskQueue.class ); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + Parameter parameter = parameterContext.getParameter(); + return LocalTaskQueue.class.equals(parameter.getType()); + } + } +} diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/DatastoreExtension.java b/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java similarity index 97% rename from java/src/test/java/com/google/appengine/tools/pipeline/DatastoreExtension.java rename to java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java index cafae64b..e074b9da 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/DatastoreExtension.java +++ b/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java @@ -1,4 +1,4 @@ -package com.google.appengine.tools.pipeline; +package com.google.appengine.tools.test; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; @@ -7,7 +7,6 @@ import org.junit.jupiter.api.extension.*; import java.net.ConnectException; -import java.time.Duration; import java.util.logging.Level; /** diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/PipelineSetupExtensions.java b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java similarity index 98% rename from java/src/test/java/com/google/appengine/tools/mapreduce/PipelineSetupExtensions.java rename to java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java index 5d518f41..2d0523d2 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/PipelineSetupExtensions.java +++ b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java @@ -1,4 +1,4 @@ -package com.google.appengine.tools.mapreduce; +package com.google.appengine.tools.test; import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunner; import com.google.appengine.tools.pipeline.*; @@ -29,6 +29,8 @@ DatastoreExtension.ParameterResolver.class, PipelineComponentsExtension.class, PipelineComponentsExtension.ParameterResolver.class, + CloudTasksExtension.class, + CloudTasksExtension.ParameterResolver.class }) public @interface PipelineSetupExtensions { From e3ad13461dfb230aaf56f7a4fb6713530da8775e Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Wed, 15 Jan 2025 15:45:45 -0800 Subject: [PATCH 2/6] use CLoudStorageExtension everywhere --- .../CloudStorageIntegrationTestHelper.java | 113 ------------- .../tools/mapreduce/CustomOutputTest.java | 5 +- .../tools/mapreduce/EndToEndTest.java | 25 +-- .../tools/mapreduce/EndToEndTestCase.java | 16 +- .../impl/GoogleCloudStorageMapOutputTest.java | 20 ++- ...gleCloudStorageLevelDbInputReaderTest.java | 24 +-- ...GoogleCloudStorageLineInputReaderTest.java | 6 +- .../GoogleCloudStorageLineInputTest.java | 5 +- .../GoogleCloudStorageLineInputTestCase.java | 22 +-- .../GoogleCloudStorageFileOutputTest.java | 31 ++-- ...entedGoogleCloudStorageFileOutputTest.java | 28 ++-- .../servlets/ShufflerServletTest.java | 20 ++- .../tools/test/CloudStorageExtension.java | 152 ++++++++++++++++++ .../tools/test/CloudStorageExtensions.java | 79 +-------- 14 files changed, 266 insertions(+), 280 deletions(-) delete mode 100644 java/src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java create mode 100644 java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java b/java/src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java deleted file mode 100644 index f41e6082..00000000 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.google.appengine.tools.mapreduce; - -import com.google.appengine.tools.development.testing.LocalServiceTestConfig; -import com.google.auth.Credentials; -import com.google.auth.oauth2.ServiceAccountCredentials; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import com.google.cloud.storage.testing.RemoteStorageHelper; -import lombok.Getter; -import lombok.SneakyThrows; - -import java.io.ByteArrayInputStream; -import java.time.Duration; -import java.util.Base64; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * sets up storage bucket for tests - * - * as of Apr 2020, no gcs emulator https://cloud.google.com/sdk/gcloud/reference/beta/emulators - * - * @see "https://googleapis.dev/java/google-cloud-storage/1.106.0/com/google/cloud/storage/testing/RemoteStorageHelper.html" - * - * eg, - * 1. create SA in target project; give it "Storage Admin" role - * 2. cat ~/Downloads/worklytics-ci-111242f427df.json | base64 - * 3. set output of that as your env variable - * - in IntelliJ, set this via RunConfigurations --> Env Variables. - * - in GitHub, set it as via repo --> Settings --> Secrets so it can be utilized in workflows - * - * q: better to have a helper class for this? analogous to - * @see com.google.appengine.tools.development.testing.LocalServiceTestHelper - */ -public class CloudStorageIntegrationTestHelper implements LocalServiceTestConfig { - - public final String KEY_ENV_VAR = "APPENGINE_MAPREDUCE_CI_SERVICE_ACCOUNT_KEY"; - - @Getter - Storage storage; - @Getter - static String bucket; - @Getter - String projectId; - @Getter - Credentials credentials; - - @Getter - String base64EncodedServiceAccountKey; - - @SneakyThrows - @Override - public void setUp() { - - String keyVar = System.getenv(KEY_ENV_VAR); - - - if (keyVar == null) { - //attempt w default credentials - credentials = StorageOptions.getDefaultInstance().getCredentials(); - - //TODO: more elegant solution? weirdness seems to happen if mix projects; credentials' project - // isn't exposed to java code via any public interface; yet bucket is created in the project - // to which the credentials default project is set - projectId = "worklytics-ci"; - //throw new IllegalStateException("Must set environment variable " + KEY_ENV_VAR + " as base64 encoded service account key to use for storage integration tests"); - } else { - base64EncodedServiceAccountKey = keyVar.trim(); - String jsonKey = new String(Base64.getDecoder().decode(base64EncodedServiceAccountKey.getBytes())); - - credentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream(jsonKey.getBytes())); - projectId = ((ServiceAccountCredentials) credentials).getProjectId(); - } - - storage = StorageOptions.newBuilder() - .setCredentials(credentials) - .setProjectId(projectId) - .build().getService(); - if (bucket == null) { - bucket = RemoteStorageHelper.generateBucketName(); - - //avoid test data being retained forever, even if bucket deletion post-test fails - storage.create(BucketInfo.newBuilder(bucket) - .setLifecycleRules(defaultTestDataLifecycle()) - .setSoftDeletePolicy(null) // no soft-deletion - .build()); - - //delete bucket at shutdown - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS); - } catch (Throwable e) { - Logger.getAnonymousLogger().log(Level.WARNING, "Failed to cleanup bucket: " + bucket); - } - })); - } - } - - List defaultTestDataLifecycle() { - return Collections.singletonList(new BucketInfo.LifecycleRule( - BucketInfo.LifecycleRule.LifecycleAction.newDeleteAction(), - BucketInfo.LifecycleRule.LifecycleCondition.newBuilder().setAge(30).build())); - } - - @Deprecated //attach delete to global runtime shutdown - @Override - public void tearDown() { - } -} diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java index aadf5e8a..7974e7a5 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java @@ -12,6 +12,7 @@ import com.google.appengine.tools.pipeline.JobInfo; import com.google.appengine.tools.pipeline.JobInfo.State; +import com.google.appengine.tools.test.CloudStorageExtension; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -98,8 +99,8 @@ public void testOutputInOrder() throws Exception { .setOutput(new CustomOutput()) .setNumReducers(17); MapReduceSettings mrSettings = new MapReduceSettings.Builder() - .setServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey()) - .setBucketName(getStorageTestHelper().getBucket()) + .setServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .setBucketName(getBucket()) .setDatastoreHost(datastore.getOptions().getHost()) .setProjectId(datastore.getOptions().getProjectId()) .setDatabaseId(datastore.getOptions().getDatabaseId()) diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java index 802b21b0..91c14535 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java @@ -37,6 +37,7 @@ import com.google.appengine.tools.mapreduce.reducers.ValueProjectionReducer; import com.google.appengine.tools.pipeline.JobRunId; import com.google.appengine.tools.pipeline.JobInfo; +import com.google.appengine.tools.test.CloudStorageExtension; import com.google.cloud.ReadChannel; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; @@ -82,11 +83,11 @@ public class EndToEndTest extends EndToEndTestCase { public void setUp() throws Exception { super.setUp(); cloudStorageFileOutputOptions = GoogleCloudStorageFileOutput.BaseOptions.defaults() - .withServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey()) - .withProjectId(getStorageTestHelper().getProjectId()); //prob not really needed .. + .withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .withProjectId(CloudStorageExtension.getProjectId()); //prob not really needed .. testSettings = new MapReduceSettings.Builder() - .setServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey()) - .setBucketName(getStorageTestHelper().getBucket()) + .setServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .setBucketName(bucket) .build(); } @@ -190,7 +191,7 @@ public void testMapOnlyJobWithSizeSegmentedOutput() throws Exception { String fileNamePattern = "MapOnlySegmentingTestShard-%04d/file-%04d"; SizeSegmentedGoogleCloudStorageFileOutput output = - new SizeSegmentedGoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), 30, fileNamePattern, mimeType, cloudStorageFileOutputOptions); + new SizeSegmentedGoogleCloudStorageFileOutput(getBucket(), 30, fileNamePattern, mimeType, cloudStorageFileOutputOptions); MarshallingOutput op = new MarshallingOutput<>(output, Marshallers.getStringMarshaller()); @@ -690,7 +691,7 @@ public void testPassThroughToString() throws Exception { final RandomLongInput input = new RandomLongInput(10, 1); input.setSeed(0L); runTest(new MapReduceSpecification.Builder<>(input, new Mod37Mapper(), ValueProjectionReducer - .create(), new StringOutput<>(",", new GoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), "Foo-%02d", "text/plain", cloudStorageFileOutputOptions))) + .create(), new StringOutput<>(",", new GoogleCloudStorageFileOutput(bucket, "Foo-%02d", "text/plain", cloudStorageFileOutputOptions))) .setKeyMarshaller(Marshallers.getStringMarshaller()) .setValueMarshaller(Marshallers.getLongMarshaller()).setJobName("TestPassThroughToString") .build(), new Verifier() { @@ -699,7 +700,7 @@ public void verify(MapReduceResult result) throws Exc assertEquals(1, result.getOutputResult().getNumFiles()); assertEquals(10, result.getCounters().getCounter(CounterNames.MAPPER_CALLS).getValue()); GcsFilename file = result.getOutputResult().getFile(0); - ReadChannel channel = getStorageTestHelper().getStorage().reader(file.asBlobId()); + ReadChannel channel = storage.reader(file.asBlobId()); BufferedReader reader = new BufferedReader(Channels.newReader(channel, US_ASCII.newDecoder(), -1)); String line = reader.readLine(); @@ -736,7 +737,7 @@ private void applyTestPassByteBufferToGcs(boolean sliceRetry) throws Exception { builder.setKeyMarshaller(Marshallers.getByteBufferMarshaller()); builder.setValueMarshaller(Marshallers.getByteBufferMarshaller()); builder.setReducer(ValueProjectionReducer.create()); - builder.setOutput(new GoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), "fileNamePattern-%04d", + builder.setOutput(new GoogleCloudStorageFileOutput(bucket, "fileNamePattern-%04d", "application/octet-stream", (GoogleCloudStorageFileOutput.Options) cloudStorageFileOutputOptions.withSupportSliceRetries(sliceRetry))); builder.setNumReducers(2); runTest(builder.build(), new Verifier() { @@ -747,8 +748,8 @@ public void verify(MapReduceResult result) throws Exc ArrayList results = new ArrayList<>(); ByteBuffer holder = ByteBuffer.allocate(8); for (GcsFilename file : result.getOutputResult().getFiles()) { - assertEquals("application/octet-stream", getStorageTestHelper().getStorage().get(file.asBlobId()).getContentType()); - try (ReadChannel reader = getStorageTestHelper().getStorage().reader(file.asBlobId())) { + assertEquals("application/octet-stream", storage.get(file.asBlobId()).getContentType()); + try (ReadChannel reader = storage.reader(file.asBlobId())) { int read = reader.read(holder); while (read != -1) { holder.rewind(); @@ -1239,7 +1240,7 @@ public void testSideOutput() throws Exception { final int SHARD_COUNT = 3; runTest(new MapReduceSpecification.Builder<>(new ConsecutiveLongInput(0, SHARD_COUNT, SHARD_COUNT), - new SideOutputMapper(getStorageTestHelper().getBucket(), cloudStorageFileOutputOptions), KeyProjectionReducer.create(), + new SideOutputMapper(bucket, cloudStorageFileOutputOptions), KeyProjectionReducer.create(), new InMemoryOutput<>()) .setKeyMarshaller(Marshallers.getSerializationMarshaller()) .setValueMarshaller(Marshallers.getVoidMarshaller()) @@ -1259,7 +1260,7 @@ public void verify(MapReduceResult>> result) throws Excep assertEquals(SHARD_COUNT, files.size()); for (GcsFilename file : files) { ByteBuffer buf = ByteBuffer.allocate(8); - try (ReadChannel ch = getStorageTestHelper().getStorage().reader(file.asBlobId())) { + try (ReadChannel ch = storage.reader(file.asBlobId())) { assertEquals(8, ch.read(buf)); assertEquals(-1, ch.read(ByteBuffer.allocate(1))); } diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java index f3249eff..8aa5df3e 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java @@ -19,8 +19,10 @@ import com.google.appengine.tools.pipeline.TestUtils; import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; import com.google.appengine.tools.pipeline.impl.servlets.TaskHandler; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.cloud.datastore.Datastore; +import com.google.cloud.storage.Storage; import com.google.common.base.CharMatcher; import lombok.Getter; @@ -41,6 +43,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@CloudStorageExtensions @PipelineSetupExtensions public abstract class EndToEndTestCase { @@ -61,19 +64,18 @@ public abstract class EndToEndTestCase { @Getter @Setter(onMethod_ = @BeforeEach) PipelineOrchestrator pipelineOrchestrator; + @Getter @Setter(onMethod_ = @BeforeEach) + Storage storage; + + @Getter + String bucket; + // will this magically have right context? private PipelineServlet pipelineServlet = new PipelineServlet(); private MapReduceServlet mrServlet = new MapReduceServlet(); - @Getter - private CloudStorageIntegrationTestHelper storageTestHelper; - @BeforeEach public void setUp() throws Exception { - // Creating files is not allowed in some test execution environments, so don't. - storageTestHelper = new CloudStorageIntegrationTestHelper(); - storageTestHelper.setUp(); - pipelineServlet.init(); mrServlet.init(); } diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java index cf5d07c4..2276ad7a 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java @@ -7,7 +7,11 @@ import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunId; import com.google.appengine.tools.mapreduce.inputs.GoogleCloudStorageLineInput; import com.google.appengine.tools.mapreduce.outputs.GoogleCloudStorageFileOutput; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; +import com.google.cloud.storage.Storage; import lombok.AllArgsConstructor; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -29,6 +33,7 @@ * Test class for {@link GoogleCloudStorageMapOutput}. * */ +@CloudStorageExtensions public class GoogleCloudStorageMapOutputTest { private static final ShardedJobRunId JOB = ShardedJobRunId.of("test-project", null, null,"JOB1"); @@ -41,20 +46,21 @@ public class GoogleCloudStorageMapOutputTest { private static final Random RND = new SecureRandom(); private final LocalServiceTestHelper helper = new LocalServiceTestHelper(); - private final CloudStorageIntegrationTestHelper cloudStorageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); @BeforeEach public void setUp() { helper.setUp(); System.setProperty(COMPONENTS_PER_COMPOSE_PROPERTY, String.valueOf(COMPONENTS_PER_COMPOSE)); - cloudStorageIntegrationTestHelper.setUp(); } + @Setter(onMethod_ = @BeforeEach) + Storage storage; + String bucket; + @AfterEach public void tearDown() { helper.tearDown(); System.clearProperty(COMPONENTS_PER_COMPOSE_PROPERTY); - cloudStorageIntegrationTestHelper.tearDown(); } @Test @@ -110,13 +116,13 @@ private static class SliceData { private void writeAndVerifyContent(SliceData... sliceData) throws IOException { GoogleCloudStorageFileOutput.BaseOptions outputOptions = GoogleCloudStorageFileOutput.BaseOptions.builder() - .serviceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()) - .projectId(cloudStorageIntegrationTestHelper.getProjectId()).build(); + .serviceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .projectId(CloudStorageExtension.getProjectId()).build(); GoogleCloudStorageLineInput.BaseOptions inputOptions = GoogleCloudStorageLineInput.BaseOptions.builder() - .serviceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()) + .serviceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) .build(); - GoogleCloudStorageMapOutput output = new GoogleCloudStorageMapOutput<>(cloudStorageIntegrationTestHelper.getBucket(), + GoogleCloudStorageMapOutput output = new GoogleCloudStorageMapOutput<>(this.bucket, JOB, KEY_MARSHALLER, VALUE_MARSHALLER, new Sharder() { private static final long serialVersionUID = 1L; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java index da0439e2..439f871d 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java @@ -2,8 +2,9 @@ import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; import com.google.appengine.tools.mapreduce.GcsFilename; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.mapreduce.impl.util.LevelDbConstants; import com.google.appengine.tools.mapreduce.impl.util.SerializationUtil; @@ -13,6 +14,8 @@ import com.google.appengine.tools.mapreduce.outputs.LevelDbOutputWriter; +import com.google.cloud.storage.Storage; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,37 +38,38 @@ /** * Tests for {@link GoogleCloudStorageLevelDbInput} */ +@CloudStorageExtensions @PipelineSetupExtensions public class GoogleCloudStorageLevelDbInputReaderTest { private static final int BLOCK_SIZE = LevelDbConstants.BLOCK_SIZE; - GcsFilename filename; - private CloudStorageIntegrationTestHelper storageHelper; private final LocalServiceTestHelper helper = new LocalServiceTestHelper( new LocalTaskQueueTestConfig()); + @Setter(onMethod_ = @BeforeEach) + Storage storage; + + String bucket; + GcsFilename filename; @BeforeEach public void setUp() throws Exception { helper.setUp(); - storageHelper = new CloudStorageIntegrationTestHelper(); - storageHelper.setUp(); - filename = new GcsFilename(storageHelper.getBucket(), "GoogleCloudStorageLevelDbInputReaderTest"); + filename = new GcsFilename(bucket, "GoogleCloudStorageLevelDbInputReaderTest"); } @AfterEach public void tearDown() throws Exception { - storageHelper.getStorage().delete(filename.asBlobId()); + storage.delete(filename.asBlobId()); helper.tearDown(); - storageHelper.tearDown(); } GoogleCloudStorageLineInput.Options inputOptions() { return GoogleCloudStorageLineInput.BaseOptions.builder() - .serviceAccountKey(storageHelper.getBase64EncodedServiceAccountKey()) + .serviceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) .bufferSize(BLOCK_SIZE * 2) .build(); } @@ -110,7 +114,7 @@ public void remove() { public void writeData(GcsFilename filename, ByteBufferGenerator gen) throws IOException { LevelDbOutputWriter writer = new GoogleCloudStorageLevelDbOutputWriter( new GoogleCloudStorageFileOutputWriter(filename, "application/leveldb", GoogleCloudStorageFileOutput.BaseOptions.defaults() - .withServiceAccountKey(storageHelper.getBase64EncodedServiceAccountKey()))); + .withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()))); writer.beginShard(); writer.beginSlice(); while (gen.hasNext()) { diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java index 70b439f6..f4c43fc2 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java @@ -2,6 +2,8 @@ import com.google.appengine.tools.mapreduce.GcsFilename; import com.google.appengine.tools.mapreduce.impl.util.SerializationUtil; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,9 +28,9 @@ public class GoogleCloudStorageLineInputReaderTest extends GoogleCloudStorageLin @BeforeEach public void prepareFile() throws Exception { - filename = new GcsFilename(cloudStorageIntegrationTestHelper.getBucket(), FILENAME); + filename = new GcsFilename(bucket, FILENAME); fileSize = createFile(filename.getObjectName(), RECORD, RECORDS_COUNT); - inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()); + inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()); } @Test diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java index b981db4a..7c4b698b 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java @@ -2,6 +2,7 @@ import com.google.appengine.tools.mapreduce.GcsFilename; import com.google.appengine.tools.mapreduce.InputReader; +import com.google.appengine.tools.test.CloudStorageExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -24,9 +25,9 @@ public class GoogleCloudStorageLineInputTest extends GoogleCloudStorageLineInput @BeforeEach public void setUpFile() throws Exception { - filename = new GcsFilename(cloudStorageIntegrationTestHelper.getBucket(), FILENAME); + filename = new GcsFilename(bucket, FILENAME); fileSize = createFile(filename.getObjectName(), RECORD, RECORDS_COUNT); - inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()); + inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()); } @Test diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java index d08f822c..0e989bcd 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java @@ -3,11 +3,13 @@ import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.cloud.WriteChannel; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; +import lombok.Getter; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -15,33 +17,35 @@ import java.nio.ByteBuffer; +@CloudStorageExtensions abstract class GoogleCloudStorageLineInputTestCase { - CloudStorageIntegrationTestHelper cloudStorageIntegrationTestHelper; - private final LocalServiceTestHelper helper = new LocalServiceTestHelper( new LocalDatastoreServiceTestConfig()); @BeforeEach public void setUp() throws Exception { helper.setUp(); - cloudStorageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - cloudStorageIntegrationTestHelper.setUp(); } + @Getter + @Setter(onMethod_ = @BeforeEach) + Storage storage; + + @Getter + String bucket; + long createFile(String filename, String record, int recordsCount) throws IOException { - Storage storage = cloudStorageIntegrationTestHelper.getStorage(); - try (WriteChannel writeChannel = storage.writer(BlobInfo.newBuilder(cloudStorageIntegrationTestHelper.getBucket(), filename).setContentType("application/bin").build())) { + try (WriteChannel writeChannel = storage.writer(BlobInfo.newBuilder(bucket, filename).setContentType("application/bin").build())) { for (int i = 0; i < recordsCount; i++) { writeChannel.write(ByteBuffer.wrap(record.getBytes())); } } - return cloudStorageIntegrationTestHelper.getStorage().get(BlobId.of(cloudStorageIntegrationTestHelper.getBucket(), filename)).getSize(); + return storage.get(BlobId.of(bucket, filename)).getSize(); } @AfterEach public void tearDown() throws Exception { helper.tearDown(); - cloudStorageIntegrationTestHelper.tearDown(); } } diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java index adf5ad8e..167d2e8a 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java @@ -1,15 +1,15 @@ package com.google.appengine.tools.mapreduce.outputs; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; import com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet; import com.google.appengine.tools.mapreduce.OutputWriter; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; -import lombok.Getter; -import org.junit.BeforeClass; -import org.junit.jupiter.api.AfterEach; +import com.google.cloud.storage.Storage; +import lombok.Setter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,6 +26,7 @@ import static com.google.cloud.MetadataConfig.getProjectId; import static org.junit.jupiter.api.Assertions.*; +@CloudStorageExtensions public class GoogleCloudStorageFileOutputTest { private static final String FILE_NAME_PATTERN = "shard-%02x"; @@ -37,29 +38,23 @@ public class GoogleCloudStorageFileOutputTest { // there will be some left over. private static final byte[] LARGE_CONTENT = new byte[(int) (1024 * 1024 * 2.5)]; - @Getter - CloudStorageIntegrationTestHelper storageIntegrationTestHelper; - @BeforeEach protected void setUp() throws Exception { - storageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - storageIntegrationTestHelper.setUp(); // Filling the large_content buffer with a non-repeating but consistent pattern. Random r = new Random(0); r.nextBytes(LARGE_CONTENT); } + @Setter(onMethod_ = @BeforeEach) + Storage storage; - @AfterEach - protected void tearDown() throws Exception { - storageIntegrationTestHelper.tearDown(); - } + String bucket; @Test public void testFilesAreWritten() throws IOException { GoogleCloudStorageFileOutput creator = - new GoogleCloudStorageFileOutput(storageIntegrationTestHelper.getBucket(), FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); + new GoogleCloudStorageFileOutput(bucket, FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); List> writers = creator.createWriters(NUM_SHARDS); assertEquals(NUM_SHARDS, writers.size()); beginShard(writers); @@ -73,7 +68,7 @@ public void testFilesAreWritten() throws IOException { GoogleCloudStorageFileSet files = creator.finish(writers); assertEquals(NUM_SHARDS, files.getNumFiles()); for (int i = 0; i < NUM_SHARDS; i++) { - Blob blob = storageIntegrationTestHelper.getStorage().get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); + Blob blob = storage.get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); assertNotNull(blob); assertEquals(SMALL_CONTENT.length, (long) blob.getSize()); assertEquals(MIME_TYPE, blob.getContentType()); @@ -98,7 +93,7 @@ public void testLargeSlicing() throws IOException, ClassNotFoundException { private void testSlicing(byte[] content) throws IOException, ClassNotFoundException { GoogleCloudStorageFileOutput creator = - new GoogleCloudStorageFileOutput(storageIntegrationTestHelper.getBucket(), FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); + new GoogleCloudStorageFileOutput(bucket, FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); List> writers = creator.createWriters(NUM_SHARDS); assertEquals(NUM_SHARDS, writers.size()); beginShard(writers); @@ -122,11 +117,11 @@ private void testSlicing(byte[] content) throws IOException, ClassNotFoundExcept expectedContent.rewind(); ByteBuffer actualContent = ByteBuffer.allocate(content.length * 2 + 1); BlobId blobId = BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName()); - Blob blob = storageIntegrationTestHelper.getStorage().get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); + Blob blob = storage.get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); assertNotNull(blob); assertEquals(expectedContent.capacity(), (long) blob.getSize()); assertEquals(MIME_TYPE, blob.getContentType()); - try (ReadableByteChannel readChannel = storageIntegrationTestHelper.getStorage().reader(blobId)) { + try (ReadableByteChannel readChannel = storage.reader(blobId)) { int read = readChannel.read(actualContent); assertEquals(read, content.length * 2); actualContent.limit(actualContent.position()); diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java index 485805e4..bb02c515 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java @@ -1,13 +1,16 @@ package com.google.appengine.tools.mapreduce.outputs; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; import com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet; import com.google.appengine.tools.mapreduce.OutputWriter; import com.google.appengine.tools.mapreduce.impl.util.SerializationUtil; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; import lombok.Getter; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,13 +24,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +@CloudStorageExtensions public class SizeSegmentedGoogleCloudStorageFileOutputTest { private final LocalServiceTestHelper helper = new LocalServiceTestHelper(); - @Getter - static CloudStorageIntegrationTestHelper cloudStorageIntegrationTestHelper; - private static final String MIME_TYPE = "application/json"; public static final String GCS_FILE_NAME_FORMAT = @@ -36,14 +37,15 @@ public class SizeSegmentedGoogleCloudStorageFileOutputTest { GoogleCloudStorageFileOutput.Options options; + @Getter + String bucket; + @Getter @Setter(onMethod_ = @BeforeEach) + Storage storage; + @BeforeEach protected void setUp() throws Exception { - helper.setUp(); - cloudStorageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - cloudStorageIntegrationTestHelper.setUp(); - options = GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()).withProjectId(cloudStorageIntegrationTestHelper.getProjectId()); - + options = GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()).withProjectId(CloudStorageExtension.getProjectId()); } @AfterEach @@ -56,7 +58,7 @@ public void testFilesWritten() throws IOException { int segmentSizeLimit = 10; String fileNamePattern = String.format(GCS_FILE_NAME_FORMAT, "testJob"); SizeSegmentedGoogleCloudStorageFileOutput segmenter = - new SizeSegmentedGoogleCloudStorageFileOutput(cloudStorageIntegrationTestHelper.getBucket(), segmentSizeLimit, fileNamePattern, + new SizeSegmentedGoogleCloudStorageFileOutput(bucket, segmentSizeLimit, fileNamePattern, MIME_TYPE, options); List> writers = segmenter.createWriters(5); List> finished = new ArrayList<>(); @@ -80,7 +82,7 @@ public void testFilesWritten() throws IOException { GoogleCloudStorageFileSet filesWritten = segmenter.finish(finished); assertEquals(15, filesWritten.getNumFiles()); for (int i = 0; i < filesWritten.getNumFiles(); i++) { - Blob blob = cloudStorageIntegrationTestHelper.getStorage().get(filesWritten.getFile(i).asBlobId()); + Blob blob = getStorage().get(filesWritten.getFile(i).asBlobId()); assertNotNull(blob); assertEquals(MIME_TYPE, blob.getContentType()); } @@ -90,7 +92,7 @@ public void testFilesWritten() throws IOException { public void testSegmentation() throws IOException { int segmentSizeLimit = 10; SizeSegmentedGoogleCloudStorageFileOutput segmenter = - new SizeSegmentedGoogleCloudStorageFileOutput(cloudStorageIntegrationTestHelper.getBucket(), segmentSizeLimit, "testJob/%%04d_%d", + new SizeSegmentedGoogleCloudStorageFileOutput(getBucket(), segmentSizeLimit, "testJob/%%04d_%d", MIME_TYPE, options); List> writers = segmenter.createWriters(5); int countFiles = 0; @@ -101,7 +103,7 @@ public void testSegmentation() throws IOException { GoogleCloudStorageFileSet filesWritten = segmenter.finish(writers); assertEquals(countFiles, filesWritten.getNumFiles()); for (int i = 0; i < filesWritten.getNumFiles(); i++) { - Blob blob = cloudStorageIntegrationTestHelper.getStorage().get(filesWritten.getFile(i).asBlobId()); + Blob blob = getStorage().get(filesWritten.getFile(i).asBlobId()); assertNotNull(blob); assertEquals(MIME_TYPE, blob.getContentType()); } diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java index 96f38d35..28971e8f 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java @@ -39,10 +39,13 @@ import com.google.appengine.tools.pipeline.PipelineService; import com.google.appengine.tools.pipeline.di.JobRunServiceComponent; import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.apphosting.api.ApiProxy; import com.google.cloud.ReadChannel; import com.google.cloud.datastore.Datastore; +import com.google.cloud.storage.Storage; import com.google.common.collect.ImmutableMap; import com.google.common.collect.TreeMultimap; @@ -70,6 +73,7 @@ /** * Tests for {@link ShufflerServlet} */ +@CloudStorageExtensions @PipelineSetupExtensions public class ShufflerServletTest { @@ -131,11 +135,11 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se } } - @Getter - CloudStorageIntegrationTestHelper storageIntegrationTestHelper; @Setter(onMethod_ = @BeforeEach) PipelineService pipelineService; + @Setter(onMethod_ = @BeforeEach) + Storage storage; @BeforeEach public void setUp(JobRunServiceComponent component, @@ -145,8 +149,6 @@ public void setUp(JobRunServiceComponent component, // Creating files is not allowed in some test execution environments, so don't. proxy.setProperty(LocalBlobstoreService.NO_STORAGE_PROPERTY, "true"); WAIT_ON.drainPermits(); - storageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - storageIntegrationTestHelper.setUp(); TaskRunner.extraParamValues = Map.of(RequestUtils.Params.DATASTORE_HOST, datastore.getOptions().getHost()); @@ -188,10 +190,12 @@ private int getQueueDepth() { .size(); } + String bucket; + @SneakyThrows @Test public void testDataIsOrdered() throws InterruptedException, IOException { - ShufflerParams shufflerParams = createParams(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey(), storageIntegrationTestHelper.getBucket(), 3, 2); + ShufflerParams shufflerParams = createParams(CloudStorageExtension.getBase64EncodedServiceAccountKey(), bucket, 3, 2); // for test purposes, give a manifest file name that's unique, yet known outside of the shuffle stage of the map reduce job // (in usual case, derived from the shuffle stage's job id, which isn't known outside) @@ -214,7 +218,7 @@ public void testDataIsOrdered() throws InterruptedException, IOException { @Test public void testJson() throws IOException { - ShufflerParams shufflerParams = createParams(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey(), storageIntegrationTestHelper.getBucket(), 3, 2); + ShufflerParams shufflerParams = createParams(CloudStorageExtension.getBase64EncodedServiceAccountKey(), bucket, 3, 2); Marshaller marshaller = Marshallers.getGenericJsonMarshaller(ShufflerParams.class); ByteBuffer bytes = marshaller.toBytes(shufflerParams); @@ -248,7 +252,7 @@ List>> validateOrdered(ShufflerParams shuf GcsFilename manifest = ShuffleMapReduce.getManifestFile(null, shufflerParams); List outputFiles; - try (ReadChannel readChannel = storageIntegrationTestHelper.getStorage().get(manifest.asBlobId()).reader()) { + try (ReadChannel readChannel = storage.get(manifest.asBlobId()).reader()) { byte[] manifestBytes = new byte[4000]; int read = readChannel.read(ByteBuffer.wrap(manifestBytes)); String manifestContent = new String(manifestBytes, 0, read, "UTF-8"); @@ -285,7 +289,7 @@ List>> validateOrdered(ShufflerParams shuf } GoogleCloudStorageFileOutput.Options outputOptions() { return GoogleCloudStorageFileOutput.BaseOptions.defaults() - .withServiceAccountKey(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey()); + .withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()); } private TreeMultimap writeInputFiles(ShufflerParams shufflerParams, diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java new file mode 100644 index 00000000..f53210c4 --- /dev/null +++ b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java @@ -0,0 +1,152 @@ +package com.google.appengine.tools.test; + + +import com.google.auth.Credentials; +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.testing.RemoteStorageHelper; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.*; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Parameter; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * as of Apr 2020, no gcs emulator https://cloud.google.com/sdk/gcloud/reference/beta/emulators + * + * @see "https://googleapis.dev/java/google-cloud-storage/1.106.0/com/google/cloud/storage/testing/RemoteStorageHelper.html" + * + * eg, + * 1. create SA in target project; give it "Storage Admin" role + * 2. cat ~/Downloads/worklytics-ci-111242f427df.json | base64 + * 3. set output of that as your env variable + * - in IntelliJ, set this via RunConfigurations --> Env Variables. + * - in GitHub, set it as via repo --> Settings --> Secrets so it can be utilized in workflows + */ +public class CloudStorageExtension implements BeforeAllCallback, BeforeEachCallback { + + public static final String KEY_ENV_VAR = "CI_SERVICE_ACCOUNT_KEY"; + + public static String getBase64EncodedServiceAccountKey() { + return System.getenv(KEY_ENV_VAR); + } + + static Optional getServiceAccountCredentials() { + return Optional.ofNullable(getBase64EncodedServiceAccountKey()) + .map(keyVar -> { + String base64EncodedServiceAccountKey = keyVar.trim(); + String jsonKey = new String(Base64.getDecoder().decode(base64EncodedServiceAccountKey.getBytes())); + try { + return ((ServiceAccountCredentials) ServiceAccountCredentials.fromStream(new ByteArrayInputStream(jsonKey.getBytes()))); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + public static String getProjectId() { + return getServiceAccountCredentials().map(ServiceAccountCredentials::getProjectId).orElse("worklytics-ci"); + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + String keyVar = getBase64EncodedServiceAccountKey(); + Credentials credentials; + String projectId; + if (keyVar == null) { + //attempt w default credentials + credentials = StorageOptions.getDefaultInstance().getCredentials(); + + //TODO: more elegant solution? weirdness seems to happen if mix projects; credentials' project + // isn't exposed to java code via any public interface; yet bucket is created in the project + // to which the credentials default project is set + projectId = "worklytics-ci"; + //throw new IllegalStateException("Must set environment variable " + KEY_ENV_VAR + " as base64 encoded service account key to use for storage integration tests"); + } else { + credentials = getServiceAccountCredentials().get(); + projectId = ((ServiceAccountCredentials) credentials).getProjectId(); + } + + Storage storage = StorageOptions.newBuilder() + .setCredentials(credentials) + .setProjectId(projectId) + .build().getService(); + + context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put(Storage.class, storage); + + String bucket = context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get("bucket", String.class); + if (bucket == null) { + bucket = RemoteStorageHelper.generateBucketName(); + context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put("bucket", bucket); + + //avoid test data being retained forever, even if bucket deletion post-test fails + storage.create(BucketInfo.newBuilder(bucket) + .setLifecycleRules(defaultTestDataLifecycle()) + .setSoftDeletePolicy(null) // no soft-deletion + .build()); + + //delete bucket at shutdown + final String bucketToDelete = bucket; + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + RemoteStorageHelper.forceDelete(storage, bucketToDelete, 5, TimeUnit.SECONDS); + } catch (Throwable e) { + Logger.getAnonymousLogger().log(Level.WARNING, "Failed to cleanup bucket: " + bucketToDelete); + } + })); + + } + } + + /** + * file any String-type parameter named 'bucket' with the random bucket name + * + * @param context the current extension context; never {@code null} + */ + @SneakyThrows + @Override + public void beforeEach(ExtensionContext context) { + String bucket = context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get("bucket", String.class); + Field f = Arrays.stream(context.getRequiredTestInstance().getClass().getDeclaredFields()) + .filter(field -> field.getType().equals(String.class) && field.getName().equals("bucket")) + .findAny() + .orElse(null); + if (f != null) { + f.setAccessible(true); + f.set(context.getRequiredTestInstance(), bucket); + } + } + + static List defaultTestDataLifecycle() { + return Collections.singletonList(new BucketInfo.LifecycleRule( + BucketInfo.LifecycleRule.LifecycleAction.newDeleteAction(), + BucketInfo.LifecycleRule.LifecycleCondition.newBuilder().setAge(30).build())); + } + + /** + * files any Storage-type parameter in the test class with the Storage instance created in the beforeAll method + */ + public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + return extensionContext.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get(Storage.class); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + Parameter parameter = parameterContext.getParameter(); + return Storage.class.equals(parameter.getType()); + } + } + +} \ No newline at end of file diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java index 20ecb377..ebdb85d4 100644 --- a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java +++ b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.extension.*; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -17,6 +18,7 @@ import java.util.Base64; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -36,80 +38,3 @@ } -class CloudStorageExtension implements BeforeAllCallback { - - public static final String KEY_ENV_VAR = "CI_SERVICE_ACCOUNT_KEY"; - - @Override - public void beforeAll(ExtensionContext context) throws Exception { - String keyVar = System.getenv(KEY_ENV_VAR); - Credentials credentials; - String projectId; - if (keyVar == null) { - //attempt w default credentials - credentials = StorageOptions.getDefaultInstance().getCredentials(); - - //TODO: more elegant solution? weirdness seems to happen if mix projects; credentials' project - // isn't exposed to java code via any public interface; yet bucket is created in the project - // to which the credentials default project is set - projectId = "worklytics-ci"; - //throw new IllegalStateException("Must set environment variable " + KEY_ENV_VAR + " as base64 encoded service account key to use for storage integration tests"); - } else { - String base64EncodedServiceAccountKey = keyVar.trim(); - String jsonKey = new String(Base64.getDecoder().decode(base64EncodedServiceAccountKey.getBytes())); - - credentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream(jsonKey.getBytes())); - projectId = ((ServiceAccountCredentials) credentials).getProjectId(); - } - - Storage storage = StorageOptions.newBuilder() - .setCredentials(credentials) - .setProjectId(projectId) - .build().getService(); - - context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put(Storage.class, storage); - - String bucket = context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get("bucket", String.class); - if (bucket == null) { - bucket = RemoteStorageHelper.generateBucketName(); - context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put("bucket", bucket); - - //avoid test data being retained forever, even if bucket deletion post-test fails - storage.create(BucketInfo.newBuilder(bucket) - .setLifecycleRules(defaultTestDataLifecycle()) - .setSoftDeletePolicy(null) // no soft-deletion - .build()); - - //delete bucket at shutdown - final String bucketToDelete = bucket; - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - RemoteStorageHelper.forceDelete(storage, bucketToDelete, 5, TimeUnit.SECONDS); - } catch (Throwable e) { - Logger.getAnonymousLogger().log(Level.WARNING, "Failed to cleanup bucket: " + bucketToDelete); - } - })); - } - } - - static List defaultTestDataLifecycle() { - return Collections.singletonList(new BucketInfo.LifecycleRule( - BucketInfo.LifecycleRule.LifecycleAction.newDeleteAction(), - BucketInfo.LifecycleRule.LifecycleCondition.newBuilder().setAge(30).build())); - } - - public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { - - @Override - public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - return extensionContext.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get(Storage.class); - } - - @Override - public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - Parameter parameter = parameterContext.getParameter(); - return Storage.class.equals(parameter.getType()); - } - } - -} From 1afb903556ada83dd9fec272e65e56ed1f906e9d Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Wed, 22 Jan 2025 12:01:35 -0800 Subject: [PATCH 3/6] fix cloud storage extension stuff --- .../tools/mapreduce/MapReduceJob.java | 1 + .../tools/mapreduce/impl/sort/SortWorker.java | 3 +-- .../tools/mapreduce/CustomOutputTest.java | 5 ++++ .../tools/mapreduce/EndToEndTest.java | 4 +++ .../tools/mapreduce/EndToEndTestCase.java | 2 -- .../tools/test/CloudStorageExtension.java | 21 +++++++++++++--- .../tools/test/CloudStorageExtensions.java | 25 +++---------------- .../tools/test/DatastoreExtension.java | 4 +++ .../tools/test/PipelineSetupExtensions.java | 4 +++ 9 files changed, 41 insertions(+), 28 deletions(-) diff --git a/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java b/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java index 86c74280..8b56d0f6 100644 --- a/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java +++ b/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java @@ -204,6 +204,7 @@ public Value> run() { new HashingSharder(getNumOutputFiles(readers.size())), GoogleCloudStorageFileOutput.BaseOptions.builder() .serviceAccountKey(settings.getServiceAccountKey()) + .projectId(settings.getProjectId()) .build() ); output.setContext(context); diff --git a/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java b/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java index cc0b73c3..34992dfe 100644 --- a/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java +++ b/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; /** * Sorts a set of keyValues by a lexicographical comparison of the bytes of the key. On beginSlice a @@ -132,7 +131,7 @@ private int getStoredSize() { } /** - * Re arranges the pointers so that they are ordered according to the order of the corresponding + * Re-arranges the pointers so that they are ordered according to the order of the corresponding * keys. */ private void sortData() { diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java index 7974e7a5..c53fdcf1 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java @@ -13,6 +13,7 @@ import com.google.appengine.tools.pipeline.JobInfo.State; import com.google.appengine.tools.test.CloudStorageExtension; +import lombok.Getter; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -25,6 +26,9 @@ */ public class CustomOutputTest extends EndToEndTestCase { + @Getter + String bucket; + @SuppressWarnings("serial") static class CustomWriter extends OutputWriter { final int id; @@ -99,6 +103,7 @@ public void testOutputInOrder() throws Exception { .setOutput(new CustomOutput()) .setNumReducers(17); MapReduceSettings mrSettings = new MapReduceSettings.Builder() + .setProjectId(CloudStorageExtension.getProjectId()) .setServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) .setBucketName(getBucket()) .setDatastoreHost(datastore.getOptions().getHost()) diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java index 91c14535..4dcc0072 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java @@ -45,6 +45,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import lombok.Getter; import lombok.RequiredArgsConstructor; import org.easymock.EasyMock; import org.junit.jupiter.api.BeforeEach; @@ -79,6 +80,9 @@ public class EndToEndTest extends EndToEndTestCase { GoogleCloudStorageFileOutput.Options cloudStorageFileOutputOptions; MapReduceSettings testSettings; + @Getter + String bucket; + @BeforeEach public void setUp() throws Exception { super.setUp(); diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java index 8aa5df3e..710b474e 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java @@ -67,8 +67,6 @@ public abstract class EndToEndTestCase { @Getter @Setter(onMethod_ = @BeforeEach) Storage storage; - @Getter - String bucket; // will this magically have right context? private PipelineServlet pipelineServlet = new PipelineServlet(); diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java index f53210c4..70ef3ed2 100644 --- a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java +++ b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java @@ -1,3 +1,7 @@ +/** + * Copyright 2025 Worklytics, Co. Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + package com.google.appengine.tools.test; @@ -8,7 +12,6 @@ import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.testing.RemoteStorageHelper; import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.*; import java.io.ByteArrayInputStream; @@ -22,9 +25,13 @@ /** * as of Apr 2020, no gcs emulator https://cloud.google.com/sdk/gcloud/reference/beta/emulators + * so this extension creates a temp bucket in target project * * @see "https://googleapis.dev/java/google-cloud-storage/1.106.0/com/google/cloud/storage/testing/RemoteStorageHelper.html" * + * Auth in one of two ways: + * 1) machine has default credentials; then will use eitther default project, or one set in env var named CI_PROJECT + * 2) provide a service account credentisl * eg, * 1. create SA in target project; give it "Storage Admin" role * 2. cat ~/Downloads/worklytics-ci-111242f427df.json | base64 @@ -34,6 +41,9 @@ */ public class CloudStorageExtension implements BeforeAllCallback, BeforeEachCallback { + public static final String CI_PROJECT_ENV_VAR = "CI_PROJECT"; + public static final String DEFAULT_PROJECT_ID = "worklytics-ci"; + public static final String KEY_ENV_VAR = "CI_SERVICE_ACCOUNT_KEY"; public static String getBase64EncodedServiceAccountKey() { @@ -54,7 +64,12 @@ static Optional getServiceAccountCredentials() { } public static String getProjectId() { - return getServiceAccountCredentials().map(ServiceAccountCredentials::getProjectId).orElse("worklytics-ci"); + return getServiceAccountCredentials().map(ServiceAccountCredentials::getProjectId) + .orElseGet(CloudStorageExtension::fallbackProiectId); + } + + static private String fallbackProiectId() { + return Optional.ofNullable(System.getenv(CI_PROJECT_ENV_VAR)).orElse(DEFAULT_PROJECT_ID); } @Override @@ -69,7 +84,7 @@ public void beforeAll(ExtensionContext context) throws Exception { //TODO: more elegant solution? weirdness seems to happen if mix projects; credentials' project // isn't exposed to java code via any public interface; yet bucket is created in the project // to which the credentials default project is set - projectId = "worklytics-ci"; + projectId = fallbackProiectId(); //throw new IllegalStateException("Must set environment variable " + KEY_ENV_VAR + " as base64 encoded service account key to use for storage integration tests"); } else { credentials = getServiceAccountCredentials().get(); diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java index ebdb85d4..0af61539 100644 --- a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java +++ b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java @@ -1,33 +1,16 @@ +/** + * Copyright 2025 Worklytics, Co. + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ package com.google.appengine.tools.test; -import com.google.auth.Credentials; -import com.google.auth.oauth2.ServiceAccountCredentials; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import com.google.cloud.storage.testing.RemoteStorageHelper; import org.junit.jupiter.api.extension.*; -import java.io.ByteArrayInputStream; -import java.io.IOException; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import java.lang.reflect.Parameter; -import java.util.Base64; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -/** - * Copyright 2025 Worklytics, Co. - * - * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 - */ @Target({ ElementType.TYPE, ElementType.METHOD }) @Retention(RetentionPolicy.RUNTIME) @ExtendWith({ diff --git a/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java b/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java index e074b9da..755a826f 100644 --- a/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java +++ b/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java @@ -1,3 +1,7 @@ +/** + * Copyright 2025 Worklytics, Co. + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ package com.google.appengine.tools.test; import com.google.cloud.datastore.Datastore; diff --git a/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java index 2d0523d2..7118eaed 100644 --- a/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java +++ b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java @@ -1,3 +1,7 @@ +/** + * Copyright 2025 Worklytics, Co. + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ package com.google.appengine.tools.test; import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunner; From b2879330fb3df02cacb0bad67ffd0ca35978b295 Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Tue, 4 Feb 2025 15:29:46 -0800 Subject: [PATCH 4/6] delete Fanout again --- .../servlets/ShufflerServletTest.java | 1 - .../tools/pipeline/FanoutTaskTest.java | 115 ------------------ 2 files changed, 116 deletions(-) delete mode 100644 java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java index 28971e8f..94c129d3 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java @@ -49,7 +49,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.TreeMultimap; -import lombok.Getter; import lombok.Setter; import lombok.SneakyThrows; import org.junit.jupiter.api.AfterEach; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java deleted file mode 100644 index 3d67538b..00000000 --- a/java/src/test/java/com/google/appengine/tools/pipeline/FanoutTaskTest.java +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2011 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -package com.google.appengine.tools.pipeline; - -import static com.google.appengine.tools.pipeline.impl.util.GUIDGenerator.USE_SIMPLE_GUIDS_FOR_DEBUGGING; - -import com.google.appengine.tools.test.DatastoreExtension; -import com.google.cloud.datastore.Entity; -import com.google.cloud.datastore.Key; -import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.pipeline.impl.QueueSettings; -import com.google.appengine.tools.pipeline.impl.model.FanoutTaskRecord; -import com.google.appengine.tools.pipeline.impl.model.JobRecord; -import com.google.appengine.tools.pipeline.impl.model.Slot; -import com.google.appengine.tools.pipeline.impl.tasks.FanoutTask; -import com.google.appengine.tools.pipeline.impl.tasks.FinalizeJobTask; -import com.google.appengine.tools.pipeline.impl.tasks.HandleSlotFilledTask; -import com.google.appengine.tools.pipeline.impl.tasks.RunJobTask; -import com.google.appengine.tools.pipeline.impl.tasks.Task; -import com.google.common.collect.ImmutableList; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - - -import java.util.List; - -/** - * @author rudominer@google.com (Mitch Rudominer) - */ -@ExtendWith(DatastoreExtension.class) -public class FanoutTaskTest extends PipelineTest { - - - private LocalServiceTestHelper helper = - new LocalServiceTestHelper(); - - private List listOfTasks; - byte[] encodedBytes; - private QueueSettings queueSettings1 = new QueueSettings(); - private QueueSettings queueSettings2 = new QueueSettings().setOnQueue("queue1"); - - @BeforeEach - public void setUp() throws Exception { - helper.setUp(); - System.setProperty(USE_SIMPLE_GUIDS_FOR_DEBUGGING, "true"); - Key key = JobRecord.key(getProjectId(), null , null, "job1"); - RunJobTask runJobTask = new RunJobTask(key, queueSettings1); - key = JobRecord.key(getProjectId(), null, null, "job2"); - RunJobTask runJobTask2 = new RunJobTask(key, queueSettings2); - key = JobRecord.key(getProjectId(), null, null, "job3"); - FinalizeJobTask finalizeJobTask = new FinalizeJobTask(key, queueSettings1); - key = Slot.key(getProjectId(), null, "", "slot1"); - HandleSlotFilledTask hsfTask = new HandleSlotFilledTask(key, queueSettings2); - listOfTasks = ImmutableList.of(runJobTask, runJobTask2, finalizeJobTask, hsfTask); - encodedBytes = FanoutTask.encodeTasks(listOfTasks); - } - - @AfterEach - public void tearDown() throws Exception { - helper.tearDown(); - } - - /** - * Tests the methods {@link FanoutTask#encodeTasks(java.util.Collection)} and - * {@link FanoutTask#decodeTasks(byte[])} - */ - @Test - public void testEncodeDecode() throws Exception { - checkBytes(encodedBytes); - } - - /** - * Tests conversion of {@link FanoutTaskRecord} to and from an {@link Entity} - */ - @Test - public void testFanoutTaskRecord() throws Exception { - Key rootJobKey = JobRecord.generateKey("dummy", "dummy", "dummy"); - FanoutTaskRecord record = new FanoutTaskRecord(rootJobKey, encodedBytes); - Entity entity = record.toEntity(); - // reconstitute entity - record = new FanoutTaskRecord(entity); - checkBytes(record.getPayload()); - } - - private void checkBytes(byte[] bytes) { - List reconstituted = FanoutTask.decodeTasks(bytes); - Assertions.assertEquals( listOfTasks.size(), reconstituted.size()); - for (int i = 0; i < listOfTasks.size(); i++) { - Task expected = listOfTasks.get(i); - Task actual = reconstituted.get(i); - assertEquals(i, expected, actual); - } - } - - private void assertEquals(int i, Task expected, Task actual) { - Assertions.assertEquals(expected.getType(), actual.getType(), "i=" + i); - Assertions.assertEquals(expected.toProperties(), actual.toProperties(), "i=" + i); - } -} From a43d73507b604eb9097d10b19826ad581aeeddf9 Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Tue, 4 Feb 2025 15:38:50 -0800 Subject: [PATCH 5/6] drop CloudTasksExtension from standard --- .../google/appengine/tools/test/PipelineSetupExtensions.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java index 7118eaed..f323bb8c 100644 --- a/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java +++ b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java @@ -33,8 +33,6 @@ DatastoreExtension.ParameterResolver.class, PipelineComponentsExtension.class, PipelineComponentsExtension.ParameterResolver.class, - CloudTasksExtension.class, - CloudTasksExtension.ParameterResolver.class }) public @interface PipelineSetupExtensions { From 3b18e10f5954ad16c8a93cd8994ee2edbda08772 Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Tue, 4 Feb 2025 15:43:58 -0800 Subject: [PATCH 6/6] drop MemcacheServiceTestConfig; not needed --- .../tools/mapreduce/impl/handlers/MapReduceServletTest.java | 3 +-- .../tools/mapreduce/servlets/ShufflerServletTest.java | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java index 2d7c3e90..9846a33b 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java @@ -23,7 +23,6 @@ import static org.easymock.EasyMock.verify; import static org.junit.jupiter.api.Assertions.fail; -import com.google.appengine.tools.development.testing.LocalMemcacheServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.MapReduceJob; @@ -57,7 +56,7 @@ public class MapReduceServletTest{ private final LocalServiceTestHelper helper = new LocalServiceTestHelper( - new LocalTaskQueueTestConfig(), new LocalMemcacheServiceTestConfig()); + new LocalTaskQueueTestConfig()); private MapReduceServlet servlet; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java index 94c129d3..572aaff6 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java @@ -22,7 +22,6 @@ import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.tools.cloudtasktest.JakartaServletInvokingTaskCallback; import com.google.appengine.tools.development.ApiProxyLocal; -import com.google.appengine.tools.development.testing.LocalMemcacheServiceTestConfig; import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; @@ -96,7 +95,6 @@ public class ShufflerServletTest { private final LocalServiceTestHelper helper = new LocalServiceTestHelper( new LocalTaskQueueTestConfig().setDisableAutoTaskExecution(false).setCallbackClass(TaskRunner.class), - new LocalMemcacheServiceTestConfig(), new LocalModulesServiceTestConfig() );