From 99077aabf4ec22ef422257b2e32b889d8693914a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Efe=20G=C3=B6kdemir?= Date: Wed, 23 Sep 2026 21:02:58 +0300 Subject: [PATCH 1/3] GH-1261: Reject out-of-range dictionary indices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Efe Gökdemir --- .../vector/dictionary/DictionaryEncoder.java | 2 +- .../arrow/vector/TestDictionaryVector.java | 32 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java index 4af1a8693f..284f02032c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java +++ b/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java @@ -165,7 +165,7 @@ static void retrieveIndexVector( for (int i = start; i < end; i++) { if (!indices.isNull(i)) { int indexAsInt = (int) indices.getValueAsLong(i); - if (indexAsInt > dictionaryCount) { + if (indexAsInt < 0 || indexAsInt >= dictionaryCount) { throw new IllegalArgumentException( "Provided dictionary does not contain value for index " + indexAsInt); } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 0945919b91..d97ce9e39c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -942,6 +943,35 @@ public void testNoMemoryLeak() { assertEquals(0, allocator.getAllocatedMemory(), "decode memory leak"); } + @Test + public void testDecodeRejectsDictionaryIndicesOutsideBounds() { + try (final IntVector indices = newVector(IntVector.class, "", Types.MinorType.INT, allocator); + final VarCharVector dictionaryVector = newVarCharVector("dict", allocator)) { + setVector(dictionaryVector, zero, one); + Dictionary dictionary = + new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); + + setVector(indices, dictionaryVector.getValueCount()); + IllegalArgumentException upperBoundException = + assertThrows( + IllegalArgumentException.class, + () -> DictionaryEncoder.decode(indices, dictionary, allocator)); + assertEquals( + "Provided dictionary does not contain value for index 2", + upperBoundException.getMessage()); + + setVector(indices, -1); + IllegalArgumentException negativeException = + assertThrows( + IllegalArgumentException.class, + () -> DictionaryEncoder.decode(indices, dictionary, allocator)); + assertEquals( + "Provided dictionary does not contain value for index -1", + negativeException.getMessage()); + } + assertEquals(0, allocator.getAllocatedMemory(), "decode memory leak"); + } + @Test public void testListNoMemoryLeak() { // Create a new value vector @@ -1053,7 +1083,7 @@ public void testStructNoMemoryLeak() { NullableStructWriter writer = indices.getWriter(); writer.allocate(); writer.start(); - writer.integer("f0").writeInt(1); + writer.integer("f0").writeInt(0); writer.integer("f1").writeInt(3); writer.end(); writer.setValueCount(1); From 4901f9855c370818a41ae1ce42ad3a077d18c364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Efe=20G=C3=B6kdemir?= Date: Sat, 3 Oct 2026 15:18:45 +0300 Subject: [PATCH 2/3] Fix struct dictionary decode bounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Efe Gökdemir --- .../dictionary/StructSubfieldEncoder.java | 3 +- .../arrow/vector/TestDictionaryVector.java | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java b/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java index 8ff152fb1c..b183bc84c4 100644 --- a/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java +++ b/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java @@ -207,7 +207,8 @@ public static StructVector decode( TransferPair transfer = dictionary.getVector().makeTransferPair(decodedChildVector); BaseIntVector indices = (BaseIntVector) childVector; - DictionaryEncoder.retrieveIndexVector(indices, transfer, valueCount, 0, valueCount); + DictionaryEncoder.retrieveIndexVector( + indices, transfer, dictionary.getVector().getValueCount(), 0, valueCount); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index d97ce9e39c..573c52a5da 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -1097,6 +1097,59 @@ public void testStructNoMemoryLeak() { assertEquals(0, allocator.getAllocatedMemory(), "struct decode memory leak"); } + @Test + public void testStructDecodeUsesDictionaryValueCount() { + try (final StructVector validIndices = StructVector.empty("valid", allocator); + final StructVector outOfRangeIndices = StructVector.empty("outOfRange", allocator); + final VarCharVector dictionaryVector = new VarCharVector("f0", allocator)) { + + setVector( + dictionaryVector, + "aa".getBytes(StandardCharsets.UTF_8), + "bb".getBytes(StandardCharsets.UTF_8)); + + DictionaryProvider.MapDictionaryProvider provider = + new DictionaryProvider.MapDictionaryProvider(); + Dictionary dictionary = + new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); + provider.put(dictionary); + + ArrowType int32 = new ArrowType.Int(32, true); + FieldType indexFieldType = new FieldType(true, int32, dictionary.getEncoding()); + validIndices.addOrGet("f0", indexFieldType, IntVector.class); + outOfRangeIndices.addOrGet("f0", indexFieldType, IntVector.class); + + NullableStructWriter validWriter = validIndices.getWriter(); + validWriter.allocate(); + validWriter.start(); + validWriter.integer("f0").writeInt(1); + validWriter.end(); + validIndices.setValueCount(1); + + try (StructVector decoded = StructSubfieldEncoder.decode(validIndices, provider, allocator)) { + assertArrayEquals( + new Object[] {new Text("bb")}, convertMapValuesToArray(decoded.getObject(0))); + } + + NullableStructWriter outOfRangeWriter = outOfRangeIndices.getWriter(); + outOfRangeWriter.allocate(); + for (int i = 0; i < 5; i++) { + outOfRangeWriter.start(); + outOfRangeWriter.integer("f0").writeInt(i == 0 ? 2 : 0); + outOfRangeWriter.end(); + } + outOfRangeIndices.setValueCount(5); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> StructSubfieldEncoder.decode(outOfRangeIndices, provider, allocator)); + assertEquals( + "Provided dictionary does not contain value for index 2", exception.getMessage()); + } + assertEquals(0, allocator.getAllocatedMemory(), "struct decode memory leak"); + } + private void testDictionary( Dictionary dictionary, ToIntBiFunction valGetter) { try (VarCharVector vector = new VarCharVector("vector", allocator)) { From 167a97ea0cd272136808a00d907da411d233b364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Efe=20G=C3=B6kdemir?= Date: Sun, 4 Oct 2026 18:08:47 +0300 Subject: [PATCH 3/3] Validate long dictionary indices before narrowing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Efe Gökdemir --- .../vector/dictionary/DictionaryEncoder.java | 8 +++--- .../arrow/vector/TestDictionaryVector.java | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java index 284f02032c..a6d490c805 100644 --- a/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java +++ b/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java @@ -164,12 +164,12 @@ static void retrieveIndexVector( BaseIntVector indices, TransferPair transfer, int dictionaryCount, int start, int end) { for (int i = start; i < end; i++) { if (!indices.isNull(i)) { - int indexAsInt = (int) indices.getValueAsLong(i); - if (indexAsInt < 0 || indexAsInt >= dictionaryCount) { + long index = indices.getValueAsLong(i); + if (index < 0 || index >= dictionaryCount) { throw new IllegalArgumentException( - "Provided dictionary does not contain value for index " + indexAsInt); + "Provided dictionary does not contain value for index " + index); } - transfer.copyValueSafe(indexAsInt, i); + transfer.copyValueSafe((int) index, i); } } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 573c52a5da..65c04433e5 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -972,6 +972,32 @@ public void testDecodeRejectsDictionaryIndicesOutsideBounds() { assertEquals(0, allocator.getAllocatedMemory(), "decode memory leak"); } + @Test + public void testDecodeRejectsBigIntDictionaryIndexOutsideBounds() { + try (final BigIntVector indices = new BigIntVector("indices", allocator); + final VarCharVector dictionaryVector = newVarCharVector("dict", allocator)) { + setVector(dictionaryVector, zero, one); + Dictionary dictionary = + new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); + + setVector(indices, 1L); + try (ValueVector decoded = DictionaryEncoder.decode(indices, dictionary, allocator)) { + assertEquals(new Text("bar"), decoded.getObject(0)); + } + + long largeIndex = 1L << 32; + setVector(indices, largeIndex); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> DictionaryEncoder.decode(indices, dictionary, allocator)); + assertEquals( + "Provided dictionary does not contain value for index " + largeIndex, + exception.getMessage()); + } + assertEquals(0, allocator.getAllocatedMemory(), "decode memory leak"); + } + @Test public void testListNoMemoryLeak() { // Create a new value vector