From 29ffa7118bff5ea0a7a361e3224be6b00ab3bb29 Mon Sep 17 00:00:00 2001 From: Shardul Mahadik Date: Sat, 10 Oct 2020 22:56:18 -0700 Subject: [PATCH 1/2] MR: Fix NPE when InputSplit.getLocations is called on mappers --- .../java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java index 7c95fbe20599..0b5180167d3a 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java @@ -74,9 +74,11 @@ public long getLength() { @Override public String[] getLocations() { - if (locations == null) { + if (locations == null && conf != null) { boolean localityPreferred = conf.getBoolean(InputFormatConfig.LOCALITY, false); locations = localityPreferred ? Util.blockLocations(task, conf) : ANYWHERE; + } else { + locations = ANYWHERE; } return locations; From 86a92d62783782b4b3eaf1054744505b6e9aeb91 Mon Sep 17 00:00:00 2001 From: Shardul Mahadik Date: Mon, 12 Oct 2020 19:38:41 -0700 Subject: [PATCH 2/2] Add comment --- .../main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java index 0b5180167d3a..0d6c44074539 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java @@ -74,6 +74,8 @@ public long getLength() { @Override public String[] getLocations() { + // The implementation of getLocations() is only meant to be used during split computation + // getLocations() won't be accurate when called on worker nodes and will always return "*" if (locations == null && conf != null) { boolean localityPreferred = conf.getBoolean(InputFormatConfig.LOCALITY, false); locations = localityPreferred ? Util.blockLocations(task, conf) : ANYWHERE;