diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 68b6a6bb7eb5..a6539d18f360 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -290,7 +290,6 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour | [ PreCommit RAT ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml) | N/A | `Run RAT PreCommit` | [![.github/workflows/beam_PreCommit_RAT.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml?query=event%3Aschedule) | | [ PreCommit Spotless ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml) | N/A | `Run Spotless PreCommit` | [![.github/workflows/beam_PreCommit_Spotless.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml?query=event%3Aschedule) | | [ PreCommit SQL ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml) | N/A |`Run SQL PreCommit`| [![.github/workflows/beam_PreCommit_SQL.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml?query=event%3Aschedule) | -| [ PreCommit SQL Java8 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java8.yml) | N/A |`Run SQL_Java8 PreCommit`| [![.github/workflows/beam_PreCommit_SQL_Java8.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java8.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java8.yml?query=event%3Aschedule) | | [ PreCommit SQL Java17 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml) | N/A |`Run SQL_Java17 PreCommit`| [![.github/workflows/beam_PreCommit_SQL_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml?query=event%3Aschedule) | | [ PreCommit Typescript ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml) | N/A |`Run Typescript PreCommit`| [![.github/workflows/beam_PreCommit_Typescript.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml?query=event%3Aschedule) | | [ PreCommit Website ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml) | N/A |`Run Website PreCommit`| [![.github/workflows/beam_PreCommit_Website.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml?query=event%3Aschedule) | @@ -368,9 +367,7 @@ PostCommit Jobs run in a schedule against master branch and generally do not get | [ PostCommit Java ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Direct JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml) | ['8','25'] |`beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Direct.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml?query=event%3Aschedule) | -| [ PostCommit Java ValidatesRunner Flink Java8 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Flink_Java8.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Flink.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml?query=event%3Aschedule) | -| [ PostCommit Java ValidatesRunner Spark Java8 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Spark_Java8.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Spark.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner SparkStructuredStreaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Twister2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Twister2.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml?query=event%3Aschedule) | diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 362eef052438..e32d1c9afb77 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -682,8 +682,8 @@ class BeamModulePlugin implements Plugin { aircompressor : "io.airlift:aircompressor:2.0.3", args4j : "args4j:args4j:2.33", auto_value_annotations : "com.google.auto.value:auto-value-annotations:$autovalue_version", - // TODO: https://github.com/apache/beam/issues/34993 after stopping supporting Java 8 - avro : "org.apache.avro:avro:1.11.4", + // TODO: upgrade post 1.12.1 once https://issues.apache.org/jira/browse/AVRO-4209 resolved + avro : "org.apache.avro:avro:1.12.0", aws_java_sdk2_apache_client : "software.amazon.awssdk:apache-client:$aws_java_sdk2_version", aws_java_sdk2_netty_client : "software.amazon.awssdk:netty-nio-client:$aws_java_sdk2_version", aws_java_sdk2_auth : "software.amazon.awssdk:auth:$aws_java_sdk2_version", diff --git a/sdks/java/extensions/avro/build.gradle b/sdks/java/extensions/avro/build.gradle index 6b24bf693dfb..718f07c03660 100644 --- a/sdks/java/extensions/avro/build.gradle +++ b/sdks/java/extensions/avro/build.gradle @@ -40,7 +40,7 @@ def avroVersions = [ '182' : "1.8.2", '192' : "1.9.2", '1102': "1.10.2", - '1120': "1.12.0", + '1113': "1.11.3", ] avroVersions.each { k, v -> @@ -73,7 +73,7 @@ dependencies { // Exclude Avro dependencies from "core" since Avro support moved to this extension exclude group: "org.apache.avro", module: "avro" } - testImplementation project(path: ":sdks:java:extensions:avro:vendored-test", configuration: "shadowTest") + testImplementation(library.java.avro + ':tests') testImplementation library.java.junit testImplementation "org.tukaani:xz:1.9" // marked as optional in avro testImplementation "com.esotericsoftware:kryo:5.6.2" // Used by Avro coder test diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java index 457265646420..99eb7f961901 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java @@ -156,6 +156,8 @@ "rawtypes" }) public class AvroUtils { + public static final String VERSION_AVRO = + org.apache.avro.Schema.class.getPackage().getImplementationVersion(); private static final ForLoadedType BYTES = new ForLoadedType(byte[].class); private static final ForLoadedType JAVA_INSTANT = new ForLoadedType(java.time.Instant.class); private static final ForLoadedType JAVA_LOCALE_DATE = diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/AvroVersionVerificationTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/AvroVersionVerificationTest.java index f9e9a54b0531..ac17afadae2a 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/AvroVersionVerificationTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/AvroVersionVerificationTest.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.extensions.avro; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; import static org.junit.Assert.assertEquals; -import org.apache.avro.Schema; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Assume; @@ -33,7 +33,6 @@ public class AvroVersionVerificationTest { public void testAvroVersion() { @Nullable String targetVer = System.getProperty("beam.target.avro.version"); Assume.assumeTrue(!Strings.isNullOrEmpty(targetVer)); - String actualVer = Schema.class.getPackage().getImplementationVersion(); - assertEquals(targetVer, actualVer); + assertEquals(targetVer, VERSION_AVRO); } } diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java index 69dfe71ee0b1..e1843d79eff5 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.extensions.avro.coders; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -131,8 +132,6 @@ public class AvroCoderTest { ImmutableList.of(AVRO_NESTED_SPECIFIC_RECORD, AVRO_NESTED_SPECIFIC_RECORD), ImmutableMap.of("k1", AVRO_NESTED_SPECIFIC_RECORD, "k2", AVRO_NESTED_SPECIFIC_RECORD)); - private static final String VERSION_AVRO = Schema.class.getPackage().getImplementationVersion(); - @DefaultCoder(AvroCoder.class) private static class Pojo { public String text; @@ -875,6 +874,7 @@ private static class StringableClass {} @Test public void testDeterminismCyclicClass() { + // Note: this test fails on Avro 1.12.1 due to https://issues.apache.org/jira/browse/AVRO-4209 assertNonDeterministic( AvroCoder.of(Cyclic.class), reasonField(Cyclic.class, "cyclicField", "appears recursively")); @@ -1164,6 +1164,7 @@ private static class NullableCyclicField { @Test public void testNullableNonDeterministicField() { + // Note: this test fails on Avro 1.12.1 due to https://issues.apache.org/jira/browse/AVRO-4209 assertNonDeterministic( AvroCoder.of(NullableCyclic.class), reasonField( diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroGeneratedUserFactory.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroGeneratedUserFactory.java index 65552b705dff..8f16141443e0 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroGeneratedUserFactory.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroGeneratedUserFactory.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.extensions.avro.io; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; + import java.lang.reflect.Constructor; -import org.apache.avro.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Create a {@link AvroGeneratedUser} instance with different constructors. */ public class AvroGeneratedUserFactory { private static final Logger LOG = LoggerFactory.getLogger(AvroGeneratedUserFactory.class); - private static final String VERSION_AVRO = Schema.class.getPackage().getImplementationVersion(); public static AvroGeneratedUser newInstance( String name, Integer favoriteNumber, String favoriteColor) { diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java index f4841f1cdc02..27d070d6920a 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.extensions.avro.io; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -74,9 +75,6 @@ /** Tests for AvroSource. */ @RunWith(JUnit4.class) public class AvroSourceTest { - private static final String VERSION_AVRO = - org.apache.avro.Schema.class.getPackage().getImplementationVersion(); - @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -560,10 +558,11 @@ public void testDatumReaderFactoryWithGenericRecord() throws Exception { AvroSource.DatumReaderFactory factory = (writer, reader) -> - new GenericDatumReader(writer, reader) { + new GenericDatumReader<>(writer, reader) { @Override - protected Object readString(Object old, Decoder in) throws IOException { - return super.readString(old, in) + "_custom"; + protected Object readString(Object old, Schema schema, Decoder in) + throws IOException { + return super.readString(old, schema, in) + "_custom"; } }; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactoryTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactoryTest.java index 52f7a6700275..19141a82148a 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactoryTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactoryTest.java @@ -22,6 +22,7 @@ import static org.apache.avro.file.DataFileConstants.NULL_CODEC; import static org.apache.avro.file.DataFileConstants.SNAPPY_CODEC; import static org.apache.avro.file.DataFileConstants.XZ_CODEC; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; import static org.junit.Assert.assertEquals; import java.io.ByteArrayInputStream; @@ -40,9 +41,6 @@ /** Tests of SerializableAvroCodecFactory. */ @RunWith(JUnit4.class) public class SerializableAvroCodecFactoryTest { - private static final String VERSION_AVRO = - org.apache.avro.Schema.class.getPackage().getImplementationVersion(); - private static final List avroCodecs = new ArrayList<>(); static { diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroConversionFactory.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroConversionFactory.java index d6d6aeaffa3f..cdf16f71a69b 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroConversionFactory.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroConversionFactory.java @@ -17,15 +17,14 @@ */ package org.apache.beam.sdk.extensions.avro.schemas; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; + import java.lang.reflect.Constructor; -import org.apache.avro.Schema; import org.joda.time.LocalDate; /** Create a {@link TestAvroConversion} instance with different constructors. */ public class TestAvroConversionFactory { - private static final String VERSION_AVRO = Schema.class.getPackage().getImplementationVersion(); - public static TestAvroConversion newInstance(LocalDate date) throws Exception { if (VERSION_AVRO.equals("1.8.2")) { Constructor constructor = TestAvroConversion.class.getDeclaredConstructor(LocalDate.class); diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroFactory.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroFactory.java index 4d52a35543e0..ab1fdee92691 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroFactory.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/TestAvroFactory.java @@ -17,11 +17,12 @@ */ package org.apache.beam.sdk.extensions.avro.schemas; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; + import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; -import org.apache.avro.Schema; import org.joda.time.DateTime; import org.joda.time.LocalDate; import org.slf4j.Logger; @@ -30,7 +31,6 @@ /** Create a {@link TestAvro} instance with different constructors. */ public class TestAvroFactory { private static final Logger LOG = LoggerFactory.getLogger(TestAvroFactory.class); - private static final String VERSION_AVRO = Schema.class.getPackage().getImplementationVersion(); public static TestAvro newInstance( Boolean boolNonNullable, diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java index 282158bec03d..6db5206c3cf6 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.extensions.avro.schemas.utils; +import static org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.VERSION_AVRO; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -86,9 +87,6 @@ public class AvroUtilsTest { private static final org.apache.avro.Schema NULL_SCHEMA = org.apache.avro.Schema.create(org.apache.avro.Schema.Type.NULL); - private static final String VERSION_AVRO = - org.apache.avro.Schema.class.getPackage().getImplementationVersion(); - private Iterable randomData(org.apache.avro.Schema schema, int maxLength) throws Exception { Iterable data; if (VERSION_AVRO.equals("1.8.2")) { diff --git a/sdks/java/extensions/avro/vendored-test/build.gradle b/sdks/java/extensions/avro/vendored-test/build.gradle deleted file mode 100644 index b0489c27a13b..000000000000 --- a/sdks/java/extensions/avro/vendored-test/build.gradle +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -plugins { id 'org.apache.beam.module' } -applyJavaNature( - automaticModuleName: 'org.apache.beam.sdk.extensions.avro', - exportJavadoc: false, - shadowClosure: { - dependencies { - include(dependency("org.apache.avro:avro:1.11.3:tests")) - } - }, -) - -configurations.all { - resolutionStrategy.force "org.apache.avro:avro:1.11.3:tests" -} - -dependencies { - testRuntimeOnly "org.apache.avro:avro:1.11.3:tests" -} - -description = "Apache Beam :: SDKs :: Java :: Extensions :: Avro :: Vendored Tests" -ext.summary = "Vendor Avro 1.11.3 tests for Beam, a workaround of Avro 1.11.4 not release test jar" diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index a6a99c52d712..4e4c15f367cb 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -35,8 +35,6 @@ applyJavaNature( configurations.runtimeClasspath { // Pin kafka-clients version due to <3.4.0 missing auth callback classes. resolutionStrategy.force 'org.apache.kafka:kafka-clients:3.9.0' - // iceberg needs avro:1.12.0 - resolutionStrategy.force 'org.apache.avro:avro:1.12.0' // force parquet-avro:1.15.2 to fix CVE-2025-46762 resolutionStrategy.force 'org.apache.parquet:parquet-avro:1.15.2' diff --git a/sdks/java/io/google-cloud-platform/expansion-service/build.gradle b/sdks/java/io/google-cloud-platform/expansion-service/build.gradle index d1f65c88bdc5..824a2659b84b 100644 --- a/sdks/java/io/google-cloud-platform/expansion-service/build.gradle +++ b/sdks/java/io/google-cloud-platform/expansion-service/build.gradle @@ -26,12 +26,6 @@ applyJavaNature( shadowClosure: {}, ) -configurations.runtimeClasspath { - // Pin avro to 1.11.4 due to https://github.com/apache/beam/issues/34968 - // cannot upgrade this to the latest version due to https://github.com/apache/beam/issues/34993 - resolutionStrategy.force 'org.apache.avro:avro:1.11.4' -} - description = "Apache Beam :: SDKs :: Java :: IO :: Google Cloud Platform :: Expansion Service" ext.summary = "Expansion service serving GCP Java IOs" diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java index 9d7bc65f5954..e834d4eef5eb 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java @@ -22,6 +22,7 @@ import com.google.pubsub.v1.Schema; import java.util.Map; +import org.apache.avro.AvroTypeException; import org.apache.avro.SchemaParseException; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.ProjectPath; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SchemaPath; @@ -262,7 +263,7 @@ public void fromPubsubSchema() { assertThrows( "'notatype' Avro type should throw an exception", - SchemaParseException.class, + AvroTypeException.class, () -> PubsubClient.fromPubsubSchema( new com.google.api.services.pubsub.model.Schema() diff --git a/settings.gradle.kts b/settings.gradle.kts index 4080206bb542..60ff1cf8ce1b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -174,7 +174,6 @@ include(":sdks:java:expansion-service:container") include(":sdks:java:expansion-service:app") include(":sdks:java:extensions:arrow") include(":sdks:java:extensions:avro") -include("sdks:java:extensions:avro:vendored-test") include(":sdks:java:extensions:euphoria") include(":sdks:java:extensions:kryo") include(":sdks:java:extensions:google-cloud-platform-core")