packages/avro
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!11 [sync] PR-9: Sync master release and fix CVE-2021-43045

Browse Source 
From: @openeuler-sync-bot 
Reviewed-by: @caodongxia 
Signed-off-by: @caodongxia
This commit is contained in:
openeuler-ci-bot 2023-12-19 12:05:57 +00:00 committed by Gitee
commit 408992e05c
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
2 changed files with 351 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
CVE-2021-43045.patch Normal file
View File

@ -0,0 +1,293 @@
From 4e1fefca493029ace961b7ef8889a3722458565a Mon Sep 17 00:00:00 2001
From: Philip Sanetra <PSanetra@users.noreply.github.com>
Date: Wed, 20 Oct 2021 12:46:10 +0200
Subject: [PATCH] AVRO-3225: AVRO-3226: Fix possible StackOverflowException and
OutOfMemoryException on invalid input
Origin:
https://lists.apache.org/thread/5fttw9vk6gd2p3b846nox7hcj5469xfd
https://github.com/apache/avro/commit/4e1fefca493029ace961b7ef8889a3722458565a
* AVRO-3225: Fix possible StackOverflowException on invalid input
* AVRO-3226: Fix possible OutOfMemoryException on invalid input
* AVRO-3226: Backport changes for netstandard2.0
(cherry picked from commit a1fce29d9675b4dd95dfee9db32cc505d0b2227c)
Signed-off-by: Ryan Skraba <ryan@skraba.com>
---
.../main/IO/BinaryDecoder.netstandard2.0.cs | 33 ++++++-
.../IO/BinaryDecoder.notnetstandard2.0.cs | 56 ++++++++---
.../src/apache/test/IO/BinaryCodecTests.cs | 95 +++++++++++++++++--
3 files changed, 163 insertions(+), 21 deletions(-)
diff --git a/lang/csharp/src/apache/main/IO/BinaryDecoder.netstandard2.0.cs b/lang/csharp/src/apache/main/IO/BinaryDecoder.netstandard2.0.cs
index 91afeb57e8e..8c6cb7e5c09 100644
--- a/lang/csharp/src/apache/main/IO/BinaryDecoder.netstandard2.0.cs
+++ b/lang/csharp/src/apache/main/IO/BinaryDecoder.netstandard2.0.cs
@@ -16,6 +16,8 @@
* limitations under the License.
*/
using System;
+using System.IO;
+using System.Text;
namespace Avro.IO
{
@@ -24,6 +26,11 @@ namespace Avro.IO
/// </content>
public partial class BinaryDecoder
{
+ /// <summary>
+ /// It is hard to find documentation about the real maximum array length in .NET Framework 4.6.1, but this seems to work :-/
+ /// </summary>
+ private const int MaxDotNetArrayLength = 0x3FFFFFFF;
+
/// <summary>
/// A float is written as 4 bytes.
/// The float is converted into a 32-bit integer using a method equivalent to
@@ -72,10 +79,28 @@ public double ReadDouble()
public string ReadString()
{
int length = ReadInt();
- byte[] buffer = new byte[length];
- //TODO: Fix this because it's lame;
- ReadFixed(buffer);
- return System.Text.Encoding.UTF8.GetString(buffer);
+
+ if (length < 0)
+ {
+ throw new AvroException("Can not deserialize a string with negative length!");
+ }
+
+ if (length > MaxDotNetArrayLength)
+ {
+ throw new AvroException("String length is not supported!");
+ }
+
+ using (var binaryReader = new BinaryReader(stream, Encoding.UTF8, true))
+ {
+ var bytes = binaryReader.ReadBytes(length);
+
+ if (bytes.Length != length)
+ {
+ throw new AvroException("Could not read as many bytes from stream as expected!");
+ }
+
+ return Encoding.UTF8.GetString(bytes);
+ }
}
private void Read(byte[] buffer, int start, int len)
diff --git a/lang/csharp/src/apache/main/IO/BinaryDecoder.notnetstandard2.0.cs b/lang/csharp/src/apache/main/IO/BinaryDecoder.notnetstandard2.0.cs
index 17bd8415a96..a3bd2174e1d 100644
--- a/lang/csharp/src/apache/main/IO/BinaryDecoder.notnetstandard2.0.cs
+++ b/lang/csharp/src/apache/main/IO/BinaryDecoder.notnetstandard2.0.cs
@@ -18,6 +18,7 @@
using System;
using System.Buffers;
using System.Buffers.Binary;
+using System.IO;
using System.Text;
namespace Avro.IO
@@ -28,6 +29,8 @@ namespace Avro.IO
public partial class BinaryDecoder
{
private const int StackallocThreshold = 256;
+ private const int MaxFastReadLength = 4096;
+ private const int MaxDotNetArrayLength = 0x7FFFFFC7;
/// <summary>
/// A float is written as 4 bytes.
@@ -63,23 +66,54 @@ public double ReadDouble()
/// <returns>String read from the stream.</returns>
public string ReadString()
{
- byte[] bufferArray = null;
-
int length = ReadInt();
- Span<byte> buffer = length <= StackallocThreshold ?
- stackalloc byte[length] :
- (bufferArray = ArrayPool<byte>.Shared.Rent(length)).AsSpan(0, length);
-
- Read(buffer);
- string result = Encoding.UTF8.GetString(buffer);
+ if (length < 0)
+ {
+ throw new AvroException("Can not deserialize a string with negative length!");
+ }
- if (bufferArray != null)
+ if (length <= MaxFastReadLength)
{
- ArrayPool<byte>.Shared.Return(bufferArray);
+ byte[] bufferArray = null;
+
+ try
+ {
+ Span<byte> buffer = length <= StackallocThreshold ?
+ stackalloc byte[length] :
+ (bufferArray = ArrayPool<byte>.Shared.Rent(length)).AsSpan(0, length);
+
+ Read(buffer);
+
+ return Encoding.UTF8.GetString(buffer);
+ }
+ finally
+ {
+ if (bufferArray != null)
+ {
+ ArrayPool<byte>.Shared.Return(bufferArray);
+ }
+ }
}
+ else
+ {
+ if (length > MaxDotNetArrayLength)
+ {
+ throw new AvroException("String length is not supported!");
+ }
- return result;
+ using (var binaryReader = new BinaryReader(stream, Encoding.UTF8, true))
+ {
+ var bytes = binaryReader.ReadBytes(length);
+
+ if (bytes.Length != length)
+ {
+ throw new AvroException("Could not read as many bytes from stream as expected!");
+ }
+
+ return Encoding.UTF8.GetString(bytes);
+ }
+ }
}
private void Read(byte[] buffer, int start, int len)
diff --git a/lang/csharp/src/apache/test/IO/BinaryCodecTests.cs b/lang/csharp/src/apache/test/IO/BinaryCodecTests.cs
index a6a1731e2d8..f894d7bfc4f 100644
--- a/lang/csharp/src/apache/test/IO/BinaryCodecTests.cs
+++ b/lang/csharp/src/apache/test/IO/BinaryCodecTests.cs
@@ -20,6 +20,7 @@
using NUnit.Framework;
using System.IO;
using System.Linq;
+using System.Text;
using Avro.IO;
namespace Avro.Test
@@ -214,23 +215,105 @@ public void TestString(string n, int overhead)
TestSkip(n, (Decoder d) => d.SkipString(), (Encoder e, string t) => e.WriteString(t), overhead + n.Length);
}
-#if NETCOREAPP3_1
+#if NETCOREAPP3_1_OR_GREATER
[Test]
- public void TestLargeString()
+ public void TestStringReadIntoArrayPool()
{
+ const int maxFastReadLength = 4096;
+
// Create a 16KB buffer in the Array Pool
var largeBufferToSeedPool = ArrayPool<byte>.Shared.Rent(2 << 14);
ArrayPool<byte>.Shared.Return(largeBufferToSeedPool);
- // Create a slightly less than 16KB buffer, which will use the 16KB buffer in the pool
- var n = string.Concat(Enumerable.Repeat("1234567890", 1600));
- var overhead = 3;
+ var n = string.Concat(Enumerable.Repeat("A", maxFastReadLength));
+ var overhead = 2;
TestRead(n, (Decoder d) => d.ReadString(), (Encoder e, string t) => e.WriteString(t), overhead + n.Length);
- TestSkip(n, (Decoder d) => d.SkipString(), (Encoder e, string t) => e.WriteString(t), overhead + n.Length);
}
+
+ [Test]
+ public void TestStringReadByBinaryReader()
+ {
+ const int overhead = 2;
+ const int maxFastReadLength = 4096;
+ const int expectedStringLength = maxFastReadLength + 1;
+ var n = string.Concat(Enumerable.Repeat("A", expectedStringLength));
+
+ TestRead(n, (Decoder d) => d.ReadString(), (Encoder e, string t) => e.WriteString(t), expectedStringLength + overhead);
+ }
+#endif
+
+ [Test]
+ public void TestInvalidInputWithNegativeStringLength()
+ {
+ using (MemoryStream iostr = new MemoryStream())
+ {
+ Encoder e = new BinaryEncoder(iostr);
+
+ e.WriteLong(-1);
+
+ iostr.Flush();
+ iostr.Position = 0;
+ Decoder d = new BinaryDecoder(iostr);
+
+ var exception = Assert.Throws<AvroException>(() => d.ReadString());
+
+ Assert.NotNull(exception);
+ Assert.AreEqual("Can not deserialize a string with negative length!", exception.Message);
+ iostr.Close();
+ }
+ }
+
+ [Test]
+ public void TestInvalidInputWithMaxIntAsStringLength()
+ {
+ using (MemoryStream iostr = new MemoryStream())
+ {
+ Encoder e = new BinaryEncoder(iostr);
+
+ e.WriteLong(int.MaxValue);
+ e.WriteBytes(Encoding.UTF8.GetBytes("SomeSmallString"));
+
+ iostr.Flush();
+ iostr.Position = 0;
+ Decoder d = new BinaryDecoder(iostr);
+
+ var exception = Assert.Throws<AvroException>(() => d.ReadString());
+
+ Assert.NotNull(exception);
+ Assert.AreEqual("String length is not supported!", exception.Message);
+ iostr.Close();
+ }
+ }
+
+ [Test]
+ public void TestInvalidInputWithMaxArrayLengthAsStringLength()
+ {
+ using (MemoryStream iostr = new MemoryStream())
+ {
+ Encoder e = new BinaryEncoder(iostr);
+
+#if NETCOREAPP3_1_OR_GREATER
+ const int maximumArrayLength = 0x7FFFFFC7;
+#else
+ const int maximumArrayLength = 0x7FFFFFFF / 2;
#endif
+ e.WriteLong(maximumArrayLength);
+ e.WriteBytes(Encoding.UTF8.GetBytes("SomeSmallString"));
+
+ iostr.Flush();
+ iostr.Position = 0;
+ Decoder d = new BinaryDecoder(iostr);
+
+ var exception = Assert.Throws<AvroException>(() => d.ReadString());
+
+ Assert.NotNull(exception);
+ Assert.AreEqual("Could not read as many bytes from stream as expected!", exception.Message);
+ iostr.Close();
+ }
+ }
+
[TestCase(0, 1)]
[TestCase(1, 1)]
[TestCase(64, 2)]

61
avro.spec
View File

@ -1,8 +1,9 @@
%define HADOOP_VERSION 3.2.1
%global debug_package %{nil}
Name: avro
Version: 1.10.2
Release: 1
Release: 4
Summary: Data serialization system
License: Apache-2.0
URL: http://avro.apache.org
@ -10,8 +11,9 @@ URL: http://avro.apache.org
Source0: https://github.com/apache/avro/archive/refs/tags/release-1.10.2.tar.gz
# file xmvn-reactor required by mvn_install to specify which jar package should be put in rpm
Source1: xmvn-reactor
Patch0: CVE-2021-43045.patch
BuildArch: noarch
ExclusiveArch: aarch64 x86_64
BuildRequires: maven maven-local java-1.8.0-openjdk-devel
Requires: java-1.8.0-openjdk
@ -31,7 +33,7 @@ Avro provides:
statically typed languages.
%prep
%setup -q -n avro-release-1.10.2
%autosetup -n avro-release-1.10.2 -p1
cp %{SOURCE1} ./.xmvn-reactor
echo `pwd` > absolute_prefix.log
sed -i 's/\//\\\//g' absolute_prefix.log
@ -53,6 +55,50 @@ pushd lang/java/trevni
mvn package -Dcheckstyle.skip=true -Dmaven.test.skip=true -Dhadoop.versio=%{HADOOP_VERSION} -P hadoop2
popd
pushd lang/java/tools/target
mkdir -p tmp
mv avro-tools-%{version}.jar tmp
cd tmp
jar -xvf avro-tools-%{version}.jar
#delete unsupported architecture dynamic lib
rm -rf aix/ppc64
rm -rf org/xerial/snappy/native/Linux/ppc64
rm -rf org/xerial/snappy/native/Linux/ppc
rm -rf org/xerial/snappy/native/Linux/s390x
rm -rf org/xerial/snappy/native/Linux/ppc64le
rm -rf org/xerial/snappy/native/SunOS/sparc
rm -rf linux/ppc64
rm -rf linux/s390x
rm -rf linux/ppc64le
%ifarch x86_64
rm -rf org/xerial/snappy/native/Linux/aarch64
rm -rf org/xerial/snappy/native/Linux/armv6
rm -rf org/xerial/snappy/native/Linux/armv7
rm -rf org/xerial/snappy/native/Linux/arm
rm -rf org/xerial/snappy/native/Linux/android-arm
rm -rf linux/aarch64
rm -rf linux/arm
%endif
%ifarch aarch64
rm -rf freebsd/i386
rm -rf linux/mips64
rm -rf linux/i386
rm -rf linux/amd64
rm -rf freebsd/amd64
rm -rf org/xerial/snappy/native/SunOS/x86_64
rm -rf org/xerial/snappy/native/SunOS/x86
rm -rf org/xerial/snappy/native/Linux/x86_64
rm -rf org/xerial/snappy/native/Linux/x86
rm -rf org/xerial/snappy/native/FreeBSD/x86_64
%endif
find . -name *.so | for line in `xargs`;do strip $line;done
rm -rf avro-tools-%{version}.jar
jar -cvf avro-tools-%{version}.jar ./*
mv avro-tools-%{version}.jar ../
cd ..
rm -rf tmp
popd
%install
%mvn_install
install -d -m 0755 %{buildroot}%{_datadir}/java/%{name}
@ -64,6 +110,15 @@ install -m 0755 lang/java/tools/target/avro-tools-1.10.2-nodeps.jar %{buildroot}
%{_datadir}/java/avro/avro-tools-nodeps.jar
%changelog
* Tue Dec 19 2023 wangkai <13474090681@163.com> - 1.10.2-4
- Fix CVE-2021-43045
* Mon Aug 21 2023 xu_ping <707078654@qq.com> - 1.10.2-3
- Fix aarch64 strip failure.
* Sat Mar 04 2023 Ge Wang <wangge20@h-partners.com> - 1.10.2-2
- Fix check strip failure
* Tue Jun 29 2021 Ge Wang <wangge20@huawei.com> - 1.10.2-1
- Init package