GH-44555: [C++][Compute] Allow casting struct to bigger nullable struct

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {

cpp/src/arrow/compute/kernels/scalar_cast_nested.cc

@@ -18,6 +18,7 @@
 // Implementation of casting to (or between) list types
 
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -340,6 +341,8 @@ struct CastFixedList {
 
 struct CastStruct {
  static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ static constexpr int kFillNullSentinel = -2;
+
  const CastOptions& options = CastState::Get(ctx);
  const auto& in_type = checked_cast<const StructType&>(*batch[0].type());
  const auto& out_type = checked_cast<const StructType&>(*out->type());
@@ -348,25 +351,46 @@ struct CastStruct {
 
  std::vector<int> fields_to_select(out_field_count, -1);
 
- int out_field_index = 0;
- for (int in_field_index = 0;
- in_field_index < in_field_count && out_field_index < out_field_count;
- ++in_field_index) {
- const auto& in_field = in_type.field(in_field_index);
+ std::set<std::string> all_in_field_names;
+ for (int in_field_index = 0; in_field_index < in_field_count; ++in_field_index) {
+ all_in_field_names.insert(in_type.field(in_field_index)->name());
+ }
+
+ for (int in_field_index = 0, out_field_index = 0;
+ out_field_index < out_field_count;) {
  const auto& out_field = out_type.field(out_field_index);
- if (in_field->name() == out_field->name()) {
- if (in_field->nullable() && !out_field->nullable()) {
- return Status::TypeError("cannot cast nullable field to non-nullable field: ",
- in_type.ToString(), " ", out_type.ToString());
+ if (in_field_index < in_field_count) {
+ const auto& in_field = in_type.field(in_field_index);
+ // If there are more in_fields check if they match the out_field.
+ if (in_field->name() == out_field->name()) {
+ if (in_field->nullable() && !out_field->nullable()) {
+ return Status::TypeError("cannot cast nullable field to non-nullable field: ",
+ in_type.ToString(), " ", out_type.ToString());
+ }
+ // Found matching in_field and out_field.
+ fields_to_select[out_field_index++] = in_field_index;
+ // Using the same in_field for multiple out_fields is not allowed.
+ in_field_index++;
+ continue;
  }
- fields_to_select[out_field_index++] = in_field_index;
  }
- }
-
- if (out_field_index < out_field_count) {
- return Status::TypeError(
- "struct fields don't match or are in the wrong order: Input fields: ",
- in_type.ToString(), " output fields: ", out_type.ToString());
+ if (all_in_field_names.count(out_field->name()) == 0 && out_field->nullable()) {
+ // Didn't match current in_field, but we can fill with null.
+ // Filling with null is only acceptable on nullable fields when there
+ // is definitely no in_field with matching name.
+
+ fields_to_select[out_field_index++] = kFillNullSentinel;
+ } else if (in_field_index < in_field_count) {
+ // Didn't match current in_field, and the we cannot fill with null, so
+ // try next in_field.
+ in_field_index++;
+ } else {
+ // Didn't match current in_field, we cannot fill with null, and there
+ // are no more in_fields to try, so fail.
+ return Status::TypeError(
+ "struct fields don't match or are in the wrong order: Input fields: ",
+ in_type.ToString(), " output fields: ", out_type.ToString());
+ }
  }
 
  const ArraySpan& in_array = batch[0].array;
@@ -378,17 +402,21 @@ struct CastStruct {
  in_array.offset, in_array.length));
  }
 
- out_field_index = 0;
+int out_field_index = 0;
  for (int field_index : fields_to_select) {
- const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
- in_array.offset, in_array.length));
  const auto& target_type = out->type()->field(out_field_index++)->type();
-
- ARROW_ASSIGN_OR_RAISE(Datum cast_values,
- Cast(values, target_type, options, ctx->exec_context()));
-
- DCHECK(cast_values.is_array());
- out_array->child_data.push_back(cast_values.array());
+ if (field_index == kFillNullSentinel) {
+ ARROW_ASSIGN_OR_RAISE(auto nulls,
+ MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
+ out_array->child_data.push_back(nulls->data());
+ } else {
+ const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
+ in_array.offset, in_array.length));
+ ARROW_ASSIGN_OR_RAISE(Datum cast_values,
+ Cast(values, target_type, options, ctx->exec_context()));
+ DCHECK(cast_values.is_array());
+ out_array->child_data.push_back(cast_values.array());
+ }
  }
 
  return Status::OK();

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -2540,6 +2540,8 @@ static void CheckStructToStructSubset(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, 49]");
  e2 = ArrayFromJSON(dest_value_type, "[19, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  ASSERT_OK_AND_ASSIGN(auto src,
  StructArray::Make({a1, b1, c1, d1, e1}, field_names));
  ASSERT_OK_AND_ASSIGN(auto dest1,
@@ -2565,34 +2567,38 @@ static void CheckStructToStructSubset(
  CheckCast(src, dest5);
 
  // field does not exist
- const auto dest6 = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6 = CastOptions::Safe(dest6);
+ ASSERT_OK_AND_ASSIGN(auto dest6,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}));
+ CheckCast(src, dest6);
+
+ const auto dest7 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7 = CastOptions::Safe(dest7);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options6));
+ Cast(src, options7));
 
  // fields in wrong order
- const auto dest7 = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8 = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7 = CastOptions::Safe(dest7);
+ const auto options8 = CastOptions::Safe(dest8);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options7));
+ Cast(src, options8));
 
  // duplicate missing field names
- const auto dest8 = arrow::struct_(
+ const auto dest9 = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8 = CastOptions::Safe(dest8);
+ const auto options9 = CastOptions::Safe(dest9);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options8));
+ Cast(src, options9));
 
  // duplicate present field names
  ASSERT_OK_AND_ASSIGN(
@@ -2639,6 +2645,8 @@ static void CheckStructToStructSubsetWithNulls(
  d2 = ArrayFromJSON(dest_value_type, "[6, 51, null]");
  e2 = ArrayFromJSON(dest_value_type, "[null, 17, 74]");
 
+ auto nulls = ArrayFromJSON(dest_value_type, "[null, null, null]");
+
  std::shared_ptr<Buffer> null_bitmap;
  BitmapFromVector<int>({0, 1, 0}, &null_bitmap);
 
@@ -2674,34 +2682,39 @@ static void CheckStructToStructSubsetWithNulls(
  CheckCast(src_null, dest5_null);
 
  // field does not exist
- const auto dest6_null = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("d", int16()),
- std::make_shared<Field>("f", int64())});
- const auto options6_null = CastOptions::Safe(dest6_null);
+ ASSERT_OK_AND_ASSIGN(
+ auto dest6_null,
+ StructArray::Make({a1, d1, nulls}, {"a", "d", "f"}, null_bitmap));
+ CheckCast(src_null, dest6_null);
+
+ const auto dest7_null = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("d", int16()),
+ std::make_shared<Field>("f", int64(), /*nullable=*/false)});
+ const auto options7_null = CastOptions::Safe(dest7_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options6_null));
+ Cast(src_null, options7_null));
 
  // fields in wrong order
- const auto dest7_null = arrow::struct_({std::make_shared<Field>("a", int8()),
+ const auto dest8_null = arrow::struct_({std::make_shared<Field>("a", int8()),
  std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("b", int64())});
- const auto options7_null = CastOptions::Safe(dest7_null);
+ const auto options8_null = CastOptions::Safe(dest8_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options7_null));
+ Cast(src_null, options8_null));
 
  // duplicate missing field names
- const auto dest8_null = arrow::struct_(
+ const auto dest9_null = arrow::struct_(
  {std::make_shared<Field>("a", int8()), std::make_shared<Field>("c", int16()),
  std::make_shared<Field>("d", int32()), std::make_shared<Field>("a", int64())});
- const auto options8_null = CastOptions::Safe(dest8_null);
+ const auto options9_null = CastOptions::Safe(dest9_null);
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src_null, options8_null));
+ Cast(src_null, options9_null));
 
  // duplicate present field values
  ASSERT_OK_AND_ASSIGN(
@@ -2737,20 +2750,26 @@ TEST(Cast, StructToStructSubsetWithNulls) {
 }
 
 TEST(Cast, StructToSameSizedButDifferentNamedStruct) {
- std::vector<std::string> field_names = {"a", "b"};
+ std::vector<std::string> src_field_names = {"a", "b"};
  std::shared_ptr<Array> a, b;
  a = ArrayFromJSON(int8(), "[1, 2]");
  b = ArrayFromJSON(int8(), "[3, 4]");
- ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+ auto nulls = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, src_field_names));
+
+ std::vector<std::string> dest1_field_names = {"c", "d"};
+ ASSERT_OK_AND_ASSIGN(auto dest1, StructArray::Make({nulls, nulls}, dest1_field_names));
+ CheckCast(src, dest1);
 
- const auto dest = arrow::struct_(
- {std::make_shared<Field>("c", int8()), std::make_shared<Field>("d", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("d", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
 }
 
 TEST(Cast, StructToBiggerStruct) {
@@ -2760,15 +2779,41 @@ TEST(Cast, StructToBiggerStruct) {
  b = ArrayFromJSON(int8(), "[3, 4]");
  ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
 
- const auto dest = arrow::struct_({std::make_shared<Field>("a", int8()),
- std::make_shared<Field>("b", int8()),
- std::make_shared<Field>("c", int8())});
- const auto options = CastOptions::Safe(dest);
+ const auto dest1 = arrow::struct_(
+ {std::make_shared<Field>("a", int8()), std::make_shared<Field>("b", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false)});
+ const auto options1 = CastOptions::Safe(dest1);
+
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
+ Cast(src, options1));
+
+ const auto dest2 =
+ arrow::struct_({std::make_shared<Field>("a", int8()),
+ std::make_shared<Field>("c", int8(), /*nullable=*/false),
+ std::make_shared<Field>("b", int8())});
+ const auto options2 = CastOptions::Safe(dest2);
 
  EXPECT_RAISES_WITH_MESSAGE_THAT(
  TypeError,
  ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- Cast(src, options));
+ Cast(src, options2));
+}
+
+TEST(Cast, StructToBiggerNullableStruct) {
+ std::vector<std::string> field_names = {"a", "b"};
+ std::shared_ptr<Array> a, b, c;
+ a = ArrayFromJSON(int8(), "[1, 2]");
+ b = ArrayFromJSON(int8(), "[3, 4]");
+ ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make({a, b}, field_names));
+
+ c = ArrayFromJSON(int8(), "[null, null]");
+ ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make({a, b, c}, {"a", "b", "c"}));
+ CheckCast(src, dest);
+
+ ASSERT_OK_AND_ASSIGN(auto dest2, StructArray::Make({a, c, b}, {"a", "c", "b"}));
+ CheckCast(src, dest2);
 }
 
 TEST(Cast, StructToDifferentNullabilityStruct) {

cpp/src/arrow/dataset/scanner_test.cc

@@ -2328,7 +2328,7 @@ DatasetAndBatches MakeNestedDataset() {
  field("b", boolean()),
  field("c", struct_({
  field("d", int64()),
- field("e", float64()),
+ field("e", int64()),
  })),
  });
  const auto physical_schema = ::arrow::schema({
@@ -2531,26 +2531,35 @@ TEST(ScanNode, MaterializationOfVirtualColumn) {
 TEST(ScanNode, MaterializationOfNestedVirtualColumn) {
  TestPlan plan;
 
- auto basic = MakeNestedDataset();
+ auto nested = MakeNestedDataset();
 
  auto options = std::make_shared<ScanOptions>();
  options->projection = Materialize({"a", "b", "c"}, /*include_aug_fields=*/true);
 
  ASSERT_OK(acero::Declaration::Sequence(
  {
- {"scan", ScanNodeOptions{basic.dataset, options}},
+ {"scan", ScanNodeOptions{nested.dataset, options}},
  {"augmented_project",
  acero::ProjectNodeOptions{
  {field_ref("a"), field_ref("b"), field_ref("c")}}},
  {"sink", acero::SinkNodeOptions{&plan.sink_gen}},
  })
  .AddToPlan(plan.get()));
 
- // TODO(ARROW-1888): allow scanner to "patch up" structs with casts
- EXPECT_FINISHES_AND_RAISES_WITH_MESSAGE_THAT(
- TypeError,
- ::testing::HasSubstr("struct fields don't match or are in the wrong order"),
- plan.Run());
+ auto expected = nested.batches;
+
+ for (auto& batch : expected) {
+ // Scan will fill in "c.d" with nulls.
+ ASSERT_OK_AND_ASSIGN(auto nulls,
+ MakeArrayOfNull(int64()->GetSharedPtr(), batch.length));
+ auto c_data = batch.values[2].array()->Copy();
+ c_data->child_data.insert(c_data->child_data.begin(), nulls->data());
+ c_data->type = nested.dataset->schema()->field(2)->type();
+ auto c_array = std::make_shared<StructArray>(c_data);
+ batch.values[2] = c_array;
+ }
+
+ ASSERT_THAT(plan.Run(), Finishes(ResultWith(UnorderedElementsAreArray(expected))));
 }
 
 TEST(ScanNode, MinimalEndToEnd) {