Skip to content

S191 cleanup junit extensions #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
eschultink wants to merge 7 commits into
base: main
Choose a base branch
from
Open

S191 cleanup junit extensions #25

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
476e07e
clean up extensions; better re-use
eschultink Jan 15, 2025
e3ad134
use CLoudStorageExtension everywhere
eschultink Jan 15, 2025
1afb903
fix cloud storage extension stuff
eschultink Jan 22, 2025
4eb0b02
merge main
eschultink Feb 4, 2025
b287933
delete Fanout again
eschultink Feb 4, 2025
a43d735
drop CloudTasksExtension from standard
eschultink Feb 4, 2025
3b18e10
drop MemcacheServiceTestConfig; not needed
eschultink Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 0 additions & 113 deletions ...src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java
View file
Open in desktop

This file was deleted.

5 changes: 3 additions & 2 deletions java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.google.appengine.tools.pipeline.JobInfo;
import com.google.appengine.tools.pipeline.JobInfo.State;

import com.google.appengine.tools.test.CloudStorageExtension;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;
Expand Down Expand Up @@ -98,8 +99,8 @@ public void testOutputInOrder() throws Exception {
.setOutput(new CustomOutput())
.setNumReducers(17);
MapReduceSettings mrSettings = new MapReduceSettings.Builder()
.setServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey())
.setBucketName(getStorageTestHelper().getBucket())
.setServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey())
.setBucketName(getBucket())
.setDatastoreHost(datastore.getOptions().getHost())
.setProjectId(datastore.getOptions().getProjectId())
.setDatabaseId(datastore.getOptions().getDatabaseId())
Expand Down
97 changes: 0 additions & 97 deletions java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java
View file
Open in desktop

This file was deleted.

26 changes: 13 additions & 13 deletions java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.google.appengine.tools.mapreduce.reducers.ValueProjectionReducer;
import com.google.appengine.tools.pipeline.JobRunId;
import com.google.appengine.tools.pipeline.JobInfo;
import com.google.appengine.tools.test.CloudStorageExtension;
import com.google.cloud.ReadChannel;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -75,19 +76,18 @@ public class EndToEndTest extends EndToEndTestCase {

private static final Logger log = Logger.getLogger(EndToEndTest.class.getName());


GoogleCloudStorageFileOutput.Options cloudStorageFileOutputOptions;
MapReduceSettings testSettings;

@BeforeEach
public void setUp() throws Exception {
super.setUp();
cloudStorageFileOutputOptions = GoogleCloudStorageFileOutput.BaseOptions.defaults()
.withServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey())
.withProjectId(getStorageTestHelper().getProjectId()); //prob not really needed ..
.withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey())
.withProjectId(CloudStorageExtension.getProjectId()); //prob not really needed ..
testSettings = new MapReduceSettings.Builder()
.setServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey())
.setBucketName(getStorageTestHelper().getBucket())
.setServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey())
.setBucketName(bucket)
.build();
}

Expand Down Expand Up @@ -191,7 +191,7 @@ public void testMapOnlyJobWithSizeSegmentedOutput() throws Exception {
String fileNamePattern = "MapOnlySegmentingTestShard-%04d/file-%04d";

SizeSegmentedGoogleCloudStorageFileOutput output =
new SizeSegmentedGoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), 30, fileNamePattern, mimeType, cloudStorageFileOutputOptions);
new SizeSegmentedGoogleCloudStorageFileOutput(getBucket(), 30, fileNamePattern, mimeType, cloudStorageFileOutputOptions);
MarshallingOutput<String, GoogleCloudStorageFileSet> op =
new MarshallingOutput<>(output, Marshallers.getStringMarshaller());

Expand Down Expand Up @@ -691,7 +691,7 @@ public void testPassThroughToString() throws Exception {
final RandomLongInput input = new RandomLongInput(10, 1);
input.setSeed(0L);
runTest(new MapReduceSpecification.Builder<>(input, new Mod37Mapper(), ValueProjectionReducer
.<String, Long>create(), new StringOutput<>(",", new GoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), "Foo-%02d", "text/plain", cloudStorageFileOutputOptions)))
.<String, Long>create(), new StringOutput<>(",", new GoogleCloudStorageFileOutput(bucket, "Foo-%02d", "text/plain", cloudStorageFileOutputOptions)))
.setKeyMarshaller(Marshallers.getStringMarshaller())
.setValueMarshaller(Marshallers.getLongMarshaller()).setJobName("TestPassThroughToString")
.build(), new Verifier<GoogleCloudStorageFileSet>() {
Expand All @@ -700,7 +700,7 @@ public void verify(MapReduceResult<GoogleCloudStorageFileSet> result) throws Exc
assertEquals(1, result.getOutputResult().getNumFiles());
assertEquals(10, result.getCounters().getCounter(CounterNames.MAPPER_CALLS).getValue());
GcsFilename file = result.getOutputResult().getFile(0);
ReadChannel channel = getStorageTestHelper().getStorage().reader(file.asBlobId());
ReadChannel channel = storage.reader(file.asBlobId());
BufferedReader reader =
new BufferedReader(Channels.newReader(channel, US_ASCII.newDecoder(), -1));
String line = reader.readLine();
Expand Down Expand Up @@ -737,7 +737,7 @@ private void applyTestPassByteBufferToGcs(boolean sliceRetry) throws Exception {
builder.setKeyMarshaller(Marshallers.getByteBufferMarshaller());
builder.setValueMarshaller(Marshallers.getByteBufferMarshaller());
builder.setReducer(ValueProjectionReducer.<ByteBuffer, ByteBuffer>create());
builder.setOutput(new GoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), "fileNamePattern-%04d",
builder.setOutput(new GoogleCloudStorageFileOutput(bucket, "fileNamePattern-%04d",
"application/octet-stream", (GoogleCloudStorageFileOutput.Options) cloudStorageFileOutputOptions.withSupportSliceRetries(sliceRetry)));
builder.setNumReducers(2);
runTest(builder.build(), new Verifier<GoogleCloudStorageFileSet>() {
Expand All @@ -748,8 +748,8 @@ public void verify(MapReduceResult<GoogleCloudStorageFileSet> result) throws Exc
ArrayList<Long> results = new ArrayList<>();
ByteBuffer holder = ByteBuffer.allocate(8);
for (GcsFilename file : result.getOutputResult().getFiles()) {
assertEquals("application/octet-stream", getStorageTestHelper().getStorage().get(file.asBlobId()).getContentType());
try (ReadChannel reader = getStorageTestHelper().getStorage().reader(file.asBlobId())) {
assertEquals("application/octet-stream", storage.get(file.asBlobId()).getContentType());
try (ReadChannel reader = storage.reader(file.asBlobId())) {
int read = reader.read(holder);
while (read != -1) {
holder.rewind();
Expand Down Expand Up @@ -1240,7 +1240,7 @@ public void testSideOutput() throws Exception {
final int SHARD_COUNT = 3;

runTest(new MapReduceSpecification.Builder<>(new ConsecutiveLongInput(0, SHARD_COUNT, SHARD_COUNT),
new SideOutputMapper(getStorageTestHelper().getBucket(), cloudStorageFileOutputOptions), KeyProjectionReducer.<GcsFilename, Void>create(),
new SideOutputMapper(bucket, cloudStorageFileOutputOptions), KeyProjectionReducer.<GcsFilename, Void>create(),
new InMemoryOutput<>())
.setKeyMarshaller(Marshallers.getSerializationMarshaller())
.setValueMarshaller(Marshallers.getVoidMarshaller())
Expand All @@ -1260,7 +1260,7 @@ public void verify(MapReduceResult<List<List<GcsFilename>>> result) throws Excep
assertEquals(SHARD_COUNT, files.size());
for (GcsFilename file : files) {
ByteBuffer buf = ByteBuffer.allocate(8);
try (ReadChannel ch = getStorageTestHelper().getStorage().reader(file.asBlobId())) {
try (ReadChannel ch = storage.reader(file.asBlobId())) {
assertEquals(8, ch.read(buf));
assertEquals(-1, ch.read(ByteBuffer.allocate(1)));
}
Expand Down
Loading