Skip to content

Commit 2fd7cf5

Browse files
committed
HADOOP-19253. Google GCS compilation fails due to VectorIO changes (#7002)
Fixes a compilation failure caused by HADOOP-19098 Restore original sortRanges() method signature, FileRange[] sortRanges(List<? extends FileRange>) This ensures that google GCS connector will compile again. It has also been marked as Stable so it is left alone The version returning List<? extends FileRange> has been renamed sortRangeList() Contributed by Steve Loughran
1 parent df08e0d commit 2fd7cf5

File tree

2 files changed

+44
-8
lines changed

2 files changed

+44
-8
lines changed

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,7 @@ public static List<? extends FileRange> validateAndSortRanges(
308308
validateRangeRequest(input.get(0));
309309
sortedRanges = input;
310310
} else {
311-
sortedRanges = sortRanges(input);
311+
sortedRanges = sortRangeList(input);
312312
FileRange prev = null;
313313
for (final FileRange current : sortedRanges) {
314314
validateRangeRequest(current);
@@ -341,12 +341,25 @@ public static List<? extends FileRange> validateAndSortRanges(
341341
* @param input input ranges.
342342
* @return a new list of the ranges, sorted by offset.
343343
*/
344-
public static List<? extends FileRange> sortRanges(List<? extends FileRange> input) {
344+
public static List<? extends FileRange> sortRangeList(List<? extends FileRange> input) {
345345
final List<? extends FileRange> l = new ArrayList<>(input);
346346
l.sort(Comparator.comparingLong(FileRange::getOffset));
347347
return l;
348348
}
349349

350+
/**
351+
* Sort the input ranges by offset; no validation is done.
352+
* <p>
353+
* This method is used externally and must be retained with
354+
* the signature unchanged.
355+
* @param input input ranges.
356+
* @return a new list of the ranges, sorted by offset.
357+
*/
358+
@InterfaceStability.Stable
359+
public static FileRange[] sortRanges(List<? extends FileRange> input) {
360+
return sortRangeList(input).toArray(new FileRange[0]);
361+
}
362+
350363
/**
351364
* Merge sorted ranges to optimize the access from the underlying file
352365
* system.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestVectoredReadUtils.java

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.nio.ByteBuffer;
2424
import java.nio.IntBuffer;
2525
import java.util.Collections;
26+
import java.util.Comparator;
2627
import java.util.List;
2728
import java.util.Optional;
2829
import java.util.concurrent.CompletableFuture;
@@ -47,6 +48,7 @@
4748
import static org.apache.hadoop.fs.VectoredReadUtils.mergeSortedRanges;
4849
import static org.apache.hadoop.fs.VectoredReadUtils.readRangeFrom;
4950
import static org.apache.hadoop.fs.VectoredReadUtils.readVectored;
51+
import static org.apache.hadoop.fs.VectoredReadUtils.sortRangeList;
5052
import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
5153
import static org.apache.hadoop.fs.VectoredReadUtils.validateAndSortRanges;
5254
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -196,7 +198,7 @@ public void testSortAndMerge() {
196198
);
197199
assertIsNotOrderedDisjoint(input, 100, 800);
198200
final List<CombinedFileRange> outputList = mergeSortedRanges(
199-
sortRanges(input), 100, 1001, 2500);
201+
sortRangeList(input), 100, 1001, 2500);
200202

201203
assertRangeListSize(outputList, 1);
202204
CombinedFileRange output = outputList.get(0);
@@ -208,7 +210,7 @@ public void testSortAndMerge() {
208210
// the minSeek doesn't allow the first two to merge
209211
assertIsNotOrderedDisjoint(input, 100, 100);
210212
final List<CombinedFileRange> list2 = mergeSortedRanges(
211-
sortRanges(input),
213+
sortRangeList(input),
212214
100, 1000, 2100);
213215
assertRangeListSize(list2, 2);
214216
assertRangeElement(list2, 0, 1000, 100);
@@ -219,7 +221,7 @@ public void testSortAndMerge() {
219221
// the maxSize doesn't allow the third range to merge
220222
assertIsNotOrderedDisjoint(input, 100, 800);
221223
final List<CombinedFileRange> list3 = mergeSortedRanges(
222-
sortRanges(input),
224+
sortRangeList(input),
223225
100, 1001, 2099);
224226
assertRangeListSize(list3, 2);
225227
CombinedFileRange range0 = list3.get(0);
@@ -240,7 +242,7 @@ public void testSortAndMerge() {
240242
// test the round up and round down (the maxSize doesn't allow any merges)
241243
assertIsNotOrderedDisjoint(input, 16, 700);
242244
final List<CombinedFileRange> list4 = mergeSortedRanges(
243-
sortRanges(input),
245+
sortRangeList(input),
244246
16, 1001, 100);
245247
assertRangeListSize(list4, 3);
246248
// range[992,1104)
@@ -273,6 +275,27 @@ private static <ELEMENT extends FileRange> void assertFileRange(
273275
.isEqualTo(length);
274276
}
275277

278+
/**
279+
* Verify that {@link VectoredReadUtils#sortRanges(List)}
280+
* returns an array matching the list sort ranges.
281+
*/
282+
@Test
283+
public void testArraySortRange() throws Throwable {
284+
List<FileRange> input = asList(
285+
createFileRange(3000, 100, "1"),
286+
createFileRange(2100, 100, null),
287+
createFileRange(1000, 100, "3")
288+
);
289+
final FileRange[] rangeArray = sortRanges(input);
290+
final List<? extends FileRange> rangeList = sortRangeList(input);
291+
Assertions.assertThat(rangeArray)
292+
.describedAs("range array from sortRanges()")
293+
.isSortedAccordingTo(Comparator.comparingLong(FileRange::getOffset));
294+
Assertions.assertThat(rangeList.toArray(new FileRange[0]))
295+
.describedAs("range from sortRangeList()")
296+
.isEqualTo(rangeArray);
297+
}
298+
276299
/**
277300
* Assert that a file range satisfies the conditions.
278301
* @param range range to validate
@@ -399,7 +422,7 @@ public void testSortAndMergeMoreCases() throws Exception {
399422
);
400423
assertIsNotOrderedDisjoint(input, 100, 800);
401424
List<CombinedFileRange> outputList = mergeSortedRanges(
402-
sortRanges(input), 1, 1001, 2500);
425+
sortRangeList(input), 1, 1001, 2500);
403426
Assertions.assertThat(outputList)
404427
.describedAs("merged range size")
405428
.hasSize(1);
@@ -411,7 +434,7 @@ public void testSortAndMergeMoreCases() throws Exception {
411434
assertOrderedDisjoint(outputList, 1, 800);
412435

413436
outputList = mergeSortedRanges(
414-
sortRanges(input), 100, 1001, 2500);
437+
sortRangeList(input), 100, 1001, 2500);
415438
assertRangeListSize(outputList, 1);
416439

417440
output = outputList.get(0);

0 commit comments

Comments
 (0)