Skip to content

GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thisisnic merged 19 commits into from
May 3, 2023
Merged

GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame #35173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
1108633
Only save default metadata
thisisnic Apr 16, 2023
3e7e5fb
Implement collect.StructArray and update as.data.frame.StructArray to…
thisisnic Apr 17, 2023
cb3a0d2
as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…
thisisnic Apr 17, 2023
12eeff3
as.data.frame.arrow_dplyr_query returns data.frames
thisisnic Apr 17, 2023
85c6dc2
Pass through additional arguments
thisisnic Apr 17, 2023
42b02d1
Call collect not as.data.frame inside reader functions
thisisnic Apr 17, 2023
dd07cda
Implement helper function for comparing metadata
thisisnic Apr 17, 2023
46ad76a
Don't use collect.ArrowTabular
thisisnic Apr 21, 2023
8d7c78c
Update expect_data_frame to convert object and expected into data.fra…
thisisnic Apr 21, 2023
ab93ba5
Rename function for clarity
thisisnic Apr 21, 2023
33e7c6f
Update test-utf to use expect_equal_data_frame
thisisnic Apr 21, 2023
4e0562f
Only remove metadata if it is definitely default metadata
thisisnic Apr 21, 2023
652952c
Update schema test so it uses an object where even default metadata i…
thisisnic Apr 21, 2023
e52d3cf
Remove expect_metadata function
thisisnic Apr 24, 2023
a1ace5f
Update Python tests to compare dfs
thisisnic Apr 25, 2023
e9d21ee
Remove default metadata and nothing else
thisisnic Apr 25, 2023
5761ffa
Update code to only remove class
thisisnic Apr 25, 2023
090b5de
Relocate collect.StructArray
thisisnic Apr 26, 2023
a4cdba6
Merge branch 'apache:main' into GH-34775_new_as_df
thisisnic May 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion r/R/array.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
as.vector(x)
as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down
3 changes: 2 additions & 1 deletion r/R/arrow-tabular.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
df <- x$to_data_frame()
apply_arrow_r_metadata(df, x$metadata$r)
out <- apply_arrow_r_metadata(df, x$metadata$r)
as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down
2 changes: 1 addition & 1 deletion r/R/csv.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
}

if (isTRUE(as_data_frame)) {
tab <- as.data.frame(tab)
tab <- collect.ArrowTabular(tab)
}

tab
Expand Down
7 changes: 6 additions & 1 deletion r/R/dplyr-collect.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
if (as_data_frame) {
as.data.frame(x, ...)
df <- x$to_data_frame()
apply_arrow_r_metadata(df, x$metadata$r)
} else {
x
}
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
# TODO: should this tryCatch move down into as_arrow_table()?
Expand Down
3 changes: 2 additions & 1 deletion r/R/dplyr.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
as.data.frame(out)
}

#' @export
Expand Down
3 changes: 2 additions & 1 deletion r/R/feather.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
)

if (isTRUE(as_data_frame)) {
out <- as.data.frame(out)
df <- out$to_data_frame()
out <- apply_arrow_r_metadata(df, out$metadata$r)
}
out
}
Expand Down
2 changes: 1 addition & 1 deletion r/R/ipc-stream.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
# https://issues.apache.org/jira/browse/ARROW-6830
out <- RecordBatchStreamReader$create(file)$read_table()
if (as_data_frame) {
out <- as.data.frame(out)
out <- collect.ArrowTabular(out)
}
out
}
2 changes: 1 addition & 1 deletion r/R/json.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
}

if (isTRUE(as_data_frame)) {
tab <- as.data.frame(tab)
tab <- collect.ArrowTabular(tab)
}
tab
}
Expand Down
9 changes: 9 additions & 0 deletions r/R/metadata.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
# drop problems attributes (most likely from readr)
x[["attributes"]][["problems"]] <- NULL

# remove the class if it's just data.frame
if (identical(x$attributes$class, "data.frame")) {
x$attributes <- x$attributes[names(x$attributes) != "class"]
if (is_empty(x$attributes)) {
x <- x[names(x) != "attributes"]
}
}

out <- serialize(x, NULL, ascii = TRUE)

# if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
expr = {
columns_metadata <- r_metadata$columns
if (is.data.frame(x)) {
# if columns metadata exists, apply it here
if (length(names(x)) && !is.null(columns_metadata)) {
for (name in intersect(names(columns_metadata), names(x))) {
x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down
2 changes: 1 addition & 1 deletion r/R/parquet.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
}

if (as_data_frame) {
tab <- as.data.frame(tab)
tab <- collect.ArrowTabular(tab)
}
tab
}
Expand Down
5 changes: 3 additions & 2 deletions r/tests/testthat/helper-expectation.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down
83 changes: 41 additions & 42 deletions r/tests/testthat/test-RecordBatch.R
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
)
expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

# input validation
expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

# Input validation
expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
batch[rep(c(FALSE, TRUE), 5), ],
tbl[c(2, 4, 6, 8, 10), ]
)
# bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
# int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

# input validation
expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

# can remove a column
batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

# can remove a column by index
batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

# can add a named column
batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

# can replace a column by index
batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
fct = factor(letters[1:10])
)
batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
b = struct(x = int32(), y = int32())
)
)
out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

# if not named, columns from tib are auto spliced
batch2 <- record_batch(a = 1:10, tib)
expect_equal(
batch2$schema,
schema(a = int32(), x = int32(), y = int32())
)
out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
batch <- record_batch(a = 1:10, b = tib, schema = schema)
expect_equal(batch$schema, schema)
out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
batch2 <- record_batch(!!!df)
expect_equal(batch1, batch2)
expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

batch3 <- record_batch(df, z = 1:10)
batch4 <- record_batch(!!!df, z = 1:10)
expect_equal(batch3, batch4)
expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
batch3,
cbind(df, data.frame(z = 1:10))
)

s <- schema(x = float64(), y = utf8())
batch5 <- record_batch(df, schema = s)
batch6 <- record_batch(!!!df, schema = s)
expect_equal(batch5, batch6)
expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

s2 <- schema(x = float64(), y = utf8(), z = int16())
batch7 <- record_batch(df, z = 1:10, schema = s2)
batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
expect_equal(batch7, batch8)
expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
batch7,
cbind(df, data.frame(z = 1:10))
)
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
record_batch(a = 1:10, b = 5),
tibble::tibble(a = 1:10, b = 5)
)
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
record_batch(a = Array$create(1:10), b = Scalar$create(5)),
tibble::tibble(a = 1:10, b = 5)
)

expect_data_frame(
expect_equal_data_frame(
record_batch(a = Array$create(1:10), b = Array$create(5)),
tibble::tibble(a = 1:10, b = 5)
)

expect_data_frame(
expect_equal_data_frame(
record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
tibble::tibble(a = 1:10, b = 5)
)
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame by thisisnic · Pull Request #35173 · apache/arrow · GitHub
Skip to content
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

thisisnic
merged 19 commits into

from

May 3, 2023
 Merged


GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame
#35173
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


19 commits


Select commit
Hold shift + click to select a range
1108633

Only save default metadata


thisisnic Apr 16, 2023
3e7e5fb

Implement collect.StructArray and update as.data.frame.StructArray to…


thisisnic Apr 17, 2023
cb3a0d2

as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…


thisisnic Apr 17, 2023
12eeff3

as.data.frame.arrow_dplyr_query returns data.frames


thisisnic Apr 17, 2023
85c6dc2

Pass through additional arguments


thisisnic Apr 17, 2023
42b02d1

Call collect not as.data.frame inside reader functions


thisisnic Apr 17, 2023
dd07cda

Implement helper function for comparing metadata


thisisnic Apr 17, 2023
46ad76a

Don't use collect.ArrowTabular


thisisnic Apr 21, 2023
8d7c78c

Update expect_data_frame to convert object and expected into data.fra…


thisisnic Apr 21, 2023
ab93ba5

Rename function for clarity


thisisnic Apr 21, 2023
33e7c6f

Update test-utf to use expect_equal_data_frame


thisisnic Apr 21, 2023
4e0562f

Only remove metadata if it is definitely default metadata


thisisnic Apr 21, 2023
652952c

Update schema test so it uses an object where even default metadata i…


thisisnic Apr 21, 2023
e52d3cf

Remove expect_metadata function


thisisnic Apr 24, 2023
a1ace5f

Update Python tests to compare dfs


thisisnic Apr 25, 2023
e9d21ee

Remove default metadata and nothing else


thisisnic Apr 25, 2023
5761ffa

Update code to only remove class


thisisnic Apr 25, 2023
090b5de

Relocate collect.StructArray


thisisnic Apr 26, 2023
a4cdba6

Merge branch 'apache:main' into GH-34775_new_as_df


thisisnic May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
r/R/array.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
 as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down

3 changes: 2 additions & 1 deletion
r/R/arrow-tabular.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 out <- apply_arrow_r_metadata(df, x$metadata$r)
 as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down

2 changes: 1 addition & 1 deletion
r/R/csv.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }

 tab
Expand Down

7 changes: 6 additions & 1 deletion
r/R/dplyr-collect.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
 if (as_data_frame) {
 as.data.frame(x, ...)
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 } else {
 x
 }
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
 # TODO: should this tryCatch move down into as_arrow_table()?
Expand Down

3 changes: 2 additions & 1 deletion
r/R/dplyr.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
 collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 as.data.frame(out)
}

#' @export
Expand Down

3 changes: 2 additions & 1 deletion
r/R/feather.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
 )

 if (isTRUE(as_data_frame)) {
 out <- as.data.frame(out)
 df <- out$to_data_frame()
 out <- apply_arrow_r_metadata(df, out$metadata$r)
 }
 out
}
Expand Down

2 changes: 1 addition & 1 deletion
r/R/ipc-stream.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
 # https://issues.apache.org/jira/browse/ARROW-6830
 out <- RecordBatchStreamReader$create(file)$read_table()
 if (as_data_frame) {
 out <- as.data.frame(out)
 out <- collect.ArrowTabular(out)
 }
 out
}

2 changes: 1 addition & 1 deletion
r/R/json.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

9 changes: 9 additions & 0 deletions
r/R/metadata.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
 # drop problems attributes (most likely from readr)
 x[["attributes"]][["problems"]] <- NULL

 # remove the class if it's just data.frame
 if (identical(x$attributes$class, "data.frame")) {
 x$attributes <- x$attributes[names(x$attributes) != "class"]
 if (is_empty(x$attributes)) {
 x <- x[names(x) != "attributes"]
 }
 }

 out <- serialize(x, NULL, ascii = TRUE)

 # if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
 expr = {
 columns_metadata <- r_metadata$columns
 if (is.data.frame(x)) {
 # if columns metadata exists, apply it here
 if (length(names(x)) && !is.null(columns_metadata)) {
 for (name in intersect(names(columns_metadata), names(x))) {
 x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down

2 changes: 1 addition & 1 deletion
r/R/parquet.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
 }

 if (as_data_frame) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

5 changes: 3 additions & 2 deletions
r/tests/testthat/helper-expectation.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
 expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down

83 changes: 41 additions & 42 deletions
r/tests/testthat/test-RecordBatch.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
 schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
 )
 expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

 # input validation
 expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
 batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

 batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

 # Input validation
 expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
 expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
 batch[rep(c(FALSE, TRUE), 5), ],
 tbl[c(2, 4, 6, 8, 10), ]
 )
 # bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
 # int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

 # input validation
 expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

 # can remove a column
 batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

 # can remove a column by index
 batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

 # can add a named column
 batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

 # can replace a column by index
 batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
 fct = factor(letters[1:10])
 )
 batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
 b = struct(x = int32(), y = int32())
 )
 )
 out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

 # if not named, columns from tib are auto spliced
 batch2 <- record_batch(a = 1:10, tib)
 expect_equal(
 batch2$schema,
 schema(a = int32(), x = int32(), y = int32())
 )
 out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
 schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
 batch <- record_batch(a = 1:10, b = tib, schema = schema)
 expect_equal(batch$schema, schema)
 out <- as.data.frame(batch)
 expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
 expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

 schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
 expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
 batch2 <- record_batch(!!!df)
 expect_equal(batch1, batch2)
 expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

 batch3 <- record_batch(df, z = 1:10)
 batch4 <- record_batch(!!!df, z = 1:10)
 expect_equal(batch3, batch4)
 expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch3,
 cbind(df, data.frame(z = 1:10))
 )

 s <- schema(x = float64(), y = utf8())
 batch5 <- record_batch(df, schema = s)
 batch6 <- record_batch(!!!df, schema = s)
 expect_equal(batch5, batch6)
 expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

 s2 <- schema(x = float64(), y = utf8(), z = int16())
 batch7 <- record_batch(df, z = 1:10, schema = s2)
 batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
 expect_equal(batch7, batch8)
 expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch7,
 cbind(df, data.frame(z = 1:10))
 )
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = 1:10, b = 5),
 tibble::tibble(a = 1:10, b = 5)
 )
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Scalar$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Array$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame by thisisnic · Pull Request #35173 · apache/arrow · GitHub
Skip to content
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

thisisnic
merged 19 commits into

from

May 3, 2023
 Merged


GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame
#35173
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


19 commits


Select commit
Hold shift + click to select a range
1108633

Only save default metadata


thisisnic Apr 16, 2023
3e7e5fb

Implement collect.StructArray and update as.data.frame.StructArray to…


thisisnic Apr 17, 2023
cb3a0d2

as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…


thisisnic Apr 17, 2023
12eeff3

as.data.frame.arrow_dplyr_query returns data.frames


thisisnic Apr 17, 2023
85c6dc2

Pass through additional arguments


thisisnic Apr 17, 2023
42b02d1

Call collect not as.data.frame inside reader functions


thisisnic Apr 17, 2023
dd07cda

Implement helper function for comparing metadata


thisisnic Apr 17, 2023
46ad76a

Don't use collect.ArrowTabular


thisisnic Apr 21, 2023
8d7c78c

Update expect_data_frame to convert object and expected into data.fra…


thisisnic Apr 21, 2023
ab93ba5

Rename function for clarity


thisisnic Apr 21, 2023
33e7c6f

Update test-utf to use expect_equal_data_frame


thisisnic Apr 21, 2023
4e0562f

Only remove metadata if it is definitely default metadata


thisisnic Apr 21, 2023
652952c

Update schema test so it uses an object where even default metadata i…


thisisnic Apr 21, 2023
e52d3cf

Remove expect_metadata function


thisisnic Apr 24, 2023
a1ace5f

Update Python tests to compare dfs


thisisnic Apr 25, 2023
e9d21ee

Remove default metadata and nothing else


thisisnic Apr 25, 2023
5761ffa

Update code to only remove class


thisisnic Apr 25, 2023
090b5de

Relocate collect.StructArray


thisisnic Apr 26, 2023
a4cdba6

Merge branch 'apache:main' into GH-34775_new_as_df


thisisnic May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
r/R/array.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
 as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down

3 changes: 2 additions & 1 deletion
r/R/arrow-tabular.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 out <- apply_arrow_r_metadata(df, x$metadata$r)
 as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down

2 changes: 1 addition & 1 deletion
r/R/csv.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }

 tab
Expand Down

7 changes: 6 additions & 1 deletion
r/R/dplyr-collect.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
 if (as_data_frame) {
 as.data.frame(x, ...)
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 } else {
 x
 }
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
 # TODO: should this tryCatch move down into as_arrow_table()?
Expand Down

3 changes: 2 additions & 1 deletion
r/R/dplyr.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
 collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 as.data.frame(out)
}

#' @export
Expand Down

3 changes: 2 additions & 1 deletion
r/R/feather.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
 )

 if (isTRUE(as_data_frame)) {
 out <- as.data.frame(out)
 df <- out$to_data_frame()
 out <- apply_arrow_r_metadata(df, out$metadata$r)
 }
 out
}
Expand Down

2 changes: 1 addition & 1 deletion
r/R/ipc-stream.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
 # https://issues.apache.org/jira/browse/ARROW-6830
 out <- RecordBatchStreamReader$create(file)$read_table()
 if (as_data_frame) {
 out <- as.data.frame(out)
 out <- collect.ArrowTabular(out)
 }
 out
}

2 changes: 1 addition & 1 deletion
r/R/json.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

9 changes: 9 additions & 0 deletions
r/R/metadata.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
 # drop problems attributes (most likely from readr)
 x[["attributes"]][["problems"]] <- NULL

 # remove the class if it's just data.frame
 if (identical(x$attributes$class, "data.frame")) {
 x$attributes <- x$attributes[names(x$attributes) != "class"]
 if (is_empty(x$attributes)) {
 x <- x[names(x) != "attributes"]
 }
 }

 out <- serialize(x, NULL, ascii = TRUE)

 # if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
 expr = {
 columns_metadata <- r_metadata$columns
 if (is.data.frame(x)) {
 # if columns metadata exists, apply it here
 if (length(names(x)) && !is.null(columns_metadata)) {
 for (name in intersect(names(columns_metadata), names(x))) {
 x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down

2 changes: 1 addition & 1 deletion
r/R/parquet.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
 }

 if (as_data_frame) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

5 changes: 3 additions & 2 deletions
r/tests/testthat/helper-expectation.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
 expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down

83 changes: 41 additions & 42 deletions
r/tests/testthat/test-RecordBatch.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
 schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
 )
 expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

 # input validation
 expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
 batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

 batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

 # Input validation
 expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
 expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
 batch[rep(c(FALSE, TRUE), 5), ],
 tbl[c(2, 4, 6, 8, 10), ]
 )
 # bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
 # int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

 # input validation
 expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

 # can remove a column
 batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

 # can remove a column by index
 batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

 # can add a named column
 batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

 # can replace a column by index
 batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
 fct = factor(letters[1:10])
 )
 batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
 b = struct(x = int32(), y = int32())
 )
 )
 out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

 # if not named, columns from tib are auto spliced
 batch2 <- record_batch(a = 1:10, tib)
 expect_equal(
 batch2$schema,
 schema(a = int32(), x = int32(), y = int32())
 )
 out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
 schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
 batch <- record_batch(a = 1:10, b = tib, schema = schema)
 expect_equal(batch$schema, schema)
 out <- as.data.frame(batch)
 expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
 expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

 schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
 expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
 batch2 <- record_batch(!!!df)
 expect_equal(batch1, batch2)
 expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

 batch3 <- record_batch(df, z = 1:10)
 batch4 <- record_batch(!!!df, z = 1:10)
 expect_equal(batch3, batch4)
 expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch3,
 cbind(df, data.frame(z = 1:10))
 )

 s <- schema(x = float64(), y = utf8())
 batch5 <- record_batch(df, schema = s)
 batch6 <- record_batch(!!!df, schema = s)
 expect_equal(batch5, batch6)
 expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

 s2 <- schema(x = float64(), y = utf8(), z = int16())
 batch7 <- record_batch(df, z = 1:10, schema = s2)
 batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
 expect_equal(batch7, batch8)
 expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch7,
 cbind(df, data.frame(z = 1:10))
 )
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = 1:10, b = 5),
 tibble::tibble(a = 1:10, b = 5)
 )
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Scalar$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Array$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame by thisisnic · Pull Request #35173 · apache/arrow · GitHub
Skip to content
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

thisisnic
merged 19 commits into

from

May 3, 2023
 Merged


GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame
#35173
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


19 commits


Select commit
Hold shift + click to select a range
1108633

Only save default metadata


thisisnic Apr 16, 2023
3e7e5fb

Implement collect.StructArray and update as.data.frame.StructArray to…


thisisnic Apr 17, 2023
cb3a0d2

as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…


thisisnic Apr 17, 2023
12eeff3

as.data.frame.arrow_dplyr_query returns data.frames


thisisnic Apr 17, 2023
85c6dc2

Pass through additional arguments


thisisnic Apr 17, 2023
42b02d1

Call collect not as.data.frame inside reader functions


thisisnic Apr 17, 2023
dd07cda

Implement helper function for comparing metadata


thisisnic Apr 17, 2023
46ad76a

Don't use collect.ArrowTabular


thisisnic Apr 21, 2023
8d7c78c

Update expect_data_frame to convert object and expected into data.fra…


thisisnic Apr 21, 2023
ab93ba5

Rename function for clarity


thisisnic Apr 21, 2023
33e7c6f

Update test-utf to use expect_equal_data_frame


thisisnic Apr 21, 2023
4e0562f

Only remove metadata if it is definitely default metadata


thisisnic Apr 21, 2023
652952c

Update schema test so it uses an object where even default metadata i…


thisisnic Apr 21, 2023
e52d3cf

Remove expect_metadata function


thisisnic Apr 24, 2023
a1ace5f

Update Python tests to compare dfs


thisisnic Apr 25, 2023
e9d21ee

Remove default metadata and nothing else


thisisnic Apr 25, 2023
5761ffa

Update code to only remove class


thisisnic Apr 25, 2023
090b5de

Relocate collect.StructArray


thisisnic Apr 26, 2023
a4cdba6

Merge branch 'apache:main' into GH-34775_new_as_df


thisisnic May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
r/R/array.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
 as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down

3 changes: 2 additions & 1 deletion
r/R/arrow-tabular.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 out <- apply_arrow_r_metadata(df, x$metadata$r)
 as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down

2 changes: 1 addition & 1 deletion
r/R/csv.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }

 tab
Expand Down

7 changes: 6 additions & 1 deletion
r/R/dplyr-collect.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
 if (as_data_frame) {
 as.data.frame(x, ...)
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 } else {
 x
 }
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
 # TODO: should this tryCatch move down into as_arrow_table()?
Expand Down

3 changes: 2 additions & 1 deletion
r/R/dplyr.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
 collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 as.data.frame(out)
}

#' @export
Expand Down

3 changes: 2 additions & 1 deletion
r/R/feather.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
 )

 if (isTRUE(as_data_frame)) {
 out <- as.data.frame(out)
 df <- out$to_data_frame()
 out <- apply_arrow_r_metadata(df, out$metadata$r)
 }
 out
}
Expand Down

2 changes: 1 addition & 1 deletion
r/R/ipc-stream.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
 # https://issues.apache.org/jira/browse/ARROW-6830
 out <- RecordBatchStreamReader$create(file)$read_table()
 if (as_data_frame) {
 out <- as.data.frame(out)
 out <- collect.ArrowTabular(out)
 }
 out
}

2 changes: 1 addition & 1 deletion
r/R/json.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

9 changes: 9 additions & 0 deletions
r/R/metadata.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
 # drop problems attributes (most likely from readr)
 x[["attributes"]][["problems"]] <- NULL

 # remove the class if it's just data.frame
 if (identical(x$attributes$class, "data.frame")) {
 x$attributes <- x$attributes[names(x$attributes) != "class"]
 if (is_empty(x$attributes)) {
 x <- x[names(x) != "attributes"]
 }
 }

 out <- serialize(x, NULL, ascii = TRUE)

 # if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
 expr = {
 columns_metadata <- r_metadata$columns
 if (is.data.frame(x)) {
 # if columns metadata exists, apply it here
 if (length(names(x)) && !is.null(columns_metadata)) {
 for (name in intersect(names(columns_metadata), names(x))) {
 x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down

2 changes: 1 addition & 1 deletion
r/R/parquet.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
 }

 if (as_data_frame) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

5 changes: 3 additions & 2 deletions
r/tests/testthat/helper-expectation.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
 expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down

83 changes: 41 additions & 42 deletions
r/tests/testthat/test-RecordBatch.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
 schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
 )
 expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

 # input validation
 expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
 batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

 batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

 # Input validation
 expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
 expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
 batch[rep(c(FALSE, TRUE), 5), ],
 tbl[c(2, 4, 6, 8, 10), ]
 )
 # bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
 # int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

 # input validation
 expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

 # can remove a column
 batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

 # can remove a column by index
 batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

 # can add a named column
 batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

 # can replace a column by index
 batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
 fct = factor(letters[1:10])
 )
 batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
 b = struct(x = int32(), y = int32())
 )
 )
 out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

 # if not named, columns from tib are auto spliced
 batch2 <- record_batch(a = 1:10, tib)
 expect_equal(
 batch2$schema,
 schema(a = int32(), x = int32(), y = int32())
 )
 out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
 schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
 batch <- record_batch(a = 1:10, b = tib, schema = schema)
 expect_equal(batch$schema, schema)
 out <- as.data.frame(batch)
 expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
 expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

 schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
 expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
 batch2 <- record_batch(!!!df)
 expect_equal(batch1, batch2)
 expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

 batch3 <- record_batch(df, z = 1:10)
 batch4 <- record_batch(!!!df, z = 1:10)
 expect_equal(batch3, batch4)
 expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch3,
 cbind(df, data.frame(z = 1:10))
 )

 s <- schema(x = float64(), y = utf8())
 batch5 <- record_batch(df, schema = s)
 batch6 <- record_batch(!!!df, schema = s)
 expect_equal(batch5, batch6)
 expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

 s2 <- schema(x = float64(), y = utf8(), z = int16())
 batch7 <- record_batch(df, z = 1:10, schema = s2)
 batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
 expect_equal(batch7, batch8)
 expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch7,
 cbind(df, data.frame(z = 1:10))
 )
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = 1:10, b = 5),
 tibble::tibble(a = 1:10, b = 5)
 )
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Scalar$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Array$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame by thisisnic · Pull Request #35173 · apache/arrow · GitHub
Skip to content
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

thisisnic
merged 19 commits into

from

May 3, 2023
 Merged


GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame
#35173
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


19 commits


Select commit
Hold shift + click to select a range
1108633

Only save default metadata


thisisnic Apr 16, 2023
3e7e5fb

Implement collect.StructArray and update as.data.frame.StructArray to…


thisisnic Apr 17, 2023
cb3a0d2

as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…


thisisnic Apr 17, 2023
12eeff3

as.data.frame.arrow_dplyr_query returns data.frames


thisisnic Apr 17, 2023
85c6dc2

Pass through additional arguments


thisisnic Apr 17, 2023
42b02d1

Call collect not as.data.frame inside reader functions


thisisnic Apr 17, 2023
dd07cda

Implement helper function for comparing metadata


thisisnic Apr 17, 2023
46ad76a

Don't use collect.ArrowTabular


thisisnic Apr 21, 2023
8d7c78c

Update expect_data_frame to convert object and expected into data.fra…


thisisnic Apr 21, 2023
ab93ba5

Rename function for clarity


thisisnic Apr 21, 2023
33e7c6f

Update test-utf to use expect_equal_data_frame


thisisnic Apr 21, 2023
4e0562f

Only remove metadata if it is definitely default metadata


thisisnic Apr 21, 2023
652952c

Update schema test so it uses an object where even default metadata i…


thisisnic Apr 21, 2023
e52d3cf

Remove expect_metadata function


thisisnic Apr 24, 2023
a1ace5f

Update Python tests to compare dfs


thisisnic Apr 25, 2023
e9d21ee

Remove default metadata and nothing else


thisisnic Apr 25, 2023
5761ffa

Update code to only remove class


thisisnic Apr 25, 2023
090b5de

Relocate collect.StructArray


thisisnic Apr 26, 2023
a4cdba6

Merge branch 'apache:main' into GH-34775_new_as_df


thisisnic May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
r/R/array.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
 as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down

3 changes: 2 additions & 1 deletion
r/R/arrow-tabular.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 out <- apply_arrow_r_metadata(df, x$metadata$r)
 as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down

2 changes: 1 addition & 1 deletion
r/R/csv.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }

 tab
Expand Down

7 changes: 6 additions & 1 deletion
r/R/dplyr-collect.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
 if (as_data_frame) {
 as.data.frame(x, ...)
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 } else {
 x
 }
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
 # TODO: should this tryCatch move down into as_arrow_table()?
Expand Down

3 changes: 2 additions & 1 deletion
r/R/dplyr.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
 collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 as.data.frame(out)
}

#' @export
Expand Down

3 changes: 2 additions & 1 deletion
r/R/feather.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
 )

 if (isTRUE(as_data_frame)) {
 out <- as.data.frame(out)
 df <- out$to_data_frame()
 out <- apply_arrow_r_metadata(df, out$metadata$r)
 }
 out
}
Expand Down

2 changes: 1 addition & 1 deletion
r/R/ipc-stream.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
 # https://issues.apache.org/jira/browse/ARROW-6830
 out <- RecordBatchStreamReader$create(file)$read_table()
 if (as_data_frame) {
 out <- as.data.frame(out)
 out <- collect.ArrowTabular(out)
 }
 out
}

2 changes: 1 addition & 1 deletion
r/R/json.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

9 changes: 9 additions & 0 deletions
r/R/metadata.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
 # drop problems attributes (most likely from readr)
 x[["attributes"]][["problems"]] <- NULL

 # remove the class if it's just data.frame
 if (identical(x$attributes$class, "data.frame")) {
 x$attributes <- x$attributes[names(x$attributes) != "class"]
 if (is_empty(x$attributes)) {
 x <- x[names(x) != "attributes"]
 }
 }

 out <- serialize(x, NULL, ascii = TRUE)

 # if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
 expr = {
 columns_metadata <- r_metadata$columns
 if (is.data.frame(x)) {
 # if columns metadata exists, apply it here
 if (length(names(x)) && !is.null(columns_metadata)) {
 for (name in intersect(names(columns_metadata), names(x))) {
 x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down

2 changes: 1 addition & 1 deletion
r/R/parquet.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
 }

 if (as_data_frame) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

5 changes: 3 additions & 2 deletions
r/tests/testthat/helper-expectation.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
 expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down

83 changes: 41 additions & 42 deletions
r/tests/testthat/test-RecordBatch.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
 schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
 )
 expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

 # input validation
 expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
 batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

 batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

 # Input validation
 expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
 expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
 batch[rep(c(FALSE, TRUE), 5), ],
 tbl[c(2, 4, 6, 8, 10), ]
 )
 # bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
 # int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

 # input validation
 expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

 # can remove a column
 batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

 # can remove a column by index
 batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

 # can add a named column
 batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

 # can replace a column by index
 batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
 fct = factor(letters[1:10])
 )
 batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
 b = struct(x = int32(), y = int32())
 )
 )
 out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

 # if not named, columns from tib are auto spliced
 batch2 <- record_batch(a = 1:10, tib)
 expect_equal(
 batch2$schema,
 schema(a = int32(), x = int32(), y = int32())
 )
 out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
 schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
 batch <- record_batch(a = 1:10, b = tib, schema = schema)
 expect_equal(batch$schema, schema)
 out <- as.data.frame(batch)
 expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
 expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

 schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
 expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
 batch2 <- record_batch(!!!df)
 expect_equal(batch1, batch2)
 expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

 batch3 <- record_batch(df, z = 1:10)
 batch4 <- record_batch(!!!df, z = 1:10)
 expect_equal(batch3, batch4)
 expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch3,
 cbind(df, data.frame(z = 1:10))
 )

 s <- schema(x = float64(), y = utf8())
 batch5 <- record_batch(df, schema = s)
 batch6 <- record_batch(!!!df, schema = s)
 expect_equal(batch5, batch6)
 expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

 s2 <- schema(x = float64(), y = utf8(), z = int16())
 batch7 <- record_batch(df, z = 1:10, schema = s2)
 batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
 expect_equal(batch7, batch8)
 expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch7,
 cbind(df, data.frame(z = 1:10))
 )
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = 1:10, b = 5),
 tibble::tibble(a = 1:10, b = 5)
 )
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Scalar$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Array$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame by thisisnic · Pull Request #35173 · apache/arrow · GitHub
Skip to content
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

thisisnic
merged 19 commits into

from

May 3, 2023
 Merged


GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame
#35173
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


19 commits


Select commit
Hold shift + click to select a range
1108633

Only save default metadata


thisisnic Apr 16, 2023
3e7e5fb

Implement collect.StructArray and update as.data.frame.StructArray to…


thisisnic Apr 17, 2023
cb3a0d2

as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…


thisisnic Apr 17, 2023
12eeff3

as.data.frame.arrow_dplyr_query returns data.frames


thisisnic Apr 17, 2023
85c6dc2

Pass through additional arguments


thisisnic Apr 17, 2023
42b02d1

Call collect not as.data.frame inside reader functions


thisisnic Apr 17, 2023
dd07cda

Implement helper function for comparing metadata


thisisnic Apr 17, 2023
46ad76a

Don't use collect.ArrowTabular


thisisnic Apr 21, 2023
8d7c78c

Update expect_data_frame to convert object and expected into data.fra…


thisisnic Apr 21, 2023
ab93ba5

Rename function for clarity


thisisnic Apr 21, 2023
33e7c6f

Update test-utf to use expect_equal_data_frame


thisisnic Apr 21, 2023
4e0562f

Only remove metadata if it is definitely default metadata


thisisnic Apr 21, 2023
652952c

Update schema test so it uses an object where even default metadata i…


thisisnic Apr 21, 2023
e52d3cf

Remove expect_metadata function


thisisnic Apr 24, 2023
a1ace5f

Update Python tests to compare dfs


thisisnic Apr 25, 2023
e9d21ee

Remove default metadata and nothing else


thisisnic Apr 25, 2023
5761ffa

Update code to only remove class


thisisnic Apr 25, 2023
090b5de

Relocate collect.StructArray


thisisnic Apr 26, 2023
a4cdba6

Merge branch 'apache:main' into GH-34775_new_as_df


thisisnic May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
r/R/array.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
 as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down

3 changes: 2 additions & 1 deletion
r/R/arrow-tabular.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 out <- apply_arrow_r_metadata(df, x$metadata$r)
 as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down

2 changes: 1 addition & 1 deletion
r/R/csv.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }

 tab
Expand Down

7 changes: 6 additions & 1 deletion
r/R/dplyr-collect.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
 if (as_data_frame) {
 as.data.frame(x, ...)
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 } else {
 x
 }
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
 # TODO: should this tryCatch move down into as_arrow_table()?
Expand Down

3 changes: 2 additions & 1 deletion
r/R/dplyr.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
 collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 as.data.frame(out)
}

#' @export
Expand Down

3 changes: 2 additions & 1 deletion
r/R/feather.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
 )

 if (isTRUE(as_data_frame)) {
 out <- as.data.frame(out)
 df <- out$to_data_frame()
 out <- apply_arrow_r_metadata(df, out$metadata$r)
 }
 out
}
Expand Down

2 changes: 1 addition & 1 deletion
r/R/ipc-stream.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
 # https://issues.apache.org/jira/browse/ARROW-6830
 out <- RecordBatchStreamReader$create(file)$read_table()
 if (as_data_frame) {
 out <- as.data.frame(out)
 out <- collect.ArrowTabular(out)
 }
 out
}

2 changes: 1 addition & 1 deletion
r/R/json.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

9 changes: 9 additions & 0 deletions
r/R/metadata.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
 # drop problems attributes (most likely from readr)
 x[["attributes"]][["problems"]] <- NULL

 # remove the class if it's just data.frame
 if (identical(x$attributes$class, "data.frame")) {
 x$attributes <- x$attributes[names(x$attributes) != "class"]
 if (is_empty(x$attributes)) {
 x <- x[names(x) != "attributes"]
 }
 }

 out <- serialize(x, NULL, ascii = TRUE)

 # if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
 expr = {
 columns_metadata <- r_metadata$columns
 if (is.data.frame(x)) {
 # if columns metadata exists, apply it here
 if (length(names(x)) && !is.null(columns_metadata)) {
 for (name in intersect(names(columns_metadata), names(x))) {
 x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down

2 changes: 1 addition & 1 deletion
r/R/parquet.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
 }

 if (as_data_frame) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

5 changes: 3 additions & 2 deletions
r/tests/testthat/helper-expectation.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
 expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down

83 changes: 41 additions & 42 deletions
r/tests/testthat/test-RecordBatch.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
 schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
 )
 expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

 # input validation
 expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
 batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

 batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

 # Input validation
 expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
 expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
 batch[rep(c(FALSE, TRUE), 5), ],
 tbl[c(2, 4, 6, 8, 10), ]
 )
 # bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
 # int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

 # input validation
 expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

 # can remove a column
 batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

 # can remove a column by index
 batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

 # can add a named column
 batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

 # can replace a column by index
 batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
 fct = factor(letters[1:10])
 )
 batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
 b = struct(x = int32(), y = int32())
 )
 )
 out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

 # if not named, columns from tib are auto spliced
 batch2 <- record_batch(a = 1:10, tib)
 expect_equal(
 batch2$schema,
 schema(a = int32(), x = int32(), y = int32())
 )
 out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
 schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
 batch <- record_batch(a = 1:10, b = tib, schema = schema)
 expect_equal(batch$schema, schema)
 out <- as.data.frame(batch)
 expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
 expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

 schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
 expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
 batch2 <- record_batch(!!!df)
 expect_equal(batch1, batch2)
 expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

 batch3 <- record_batch(df, z = 1:10)
 batch4 <- record_batch(!!!df, z = 1:10)
 expect_equal(batch3, batch4)
 expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch3,
 cbind(df, data.frame(z = 1:10))
 )

 s <- schema(x = float64(), y = utf8())
 batch5 <- record_batch(df, schema = s)
 batch6 <- record_batch(!!!df, schema = s)
 expect_equal(batch5, batch6)
 expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

 s2 <- schema(x = float64(), y = utf8(), z = int16())
 batch7 <- record_batch(df, z = 1:10, schema = s2)
 batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
 expect_equal(batch7, batch8)
 expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch7,
 cbind(df, data.frame(z = 1:10))
 )
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = 1:10, b = 5),
 tibble::tibble(a = 1:10, b = 5)
 )
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Scalar$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Array$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame by thisisnic · Pull Request #35173 · apache/arrow · GitHub
Skip to content
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

thisisnic
merged 19 commits into

from

May 3, 2023
 Merged


GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame
#35173
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


19 commits


Select commit
Hold shift + click to select a range
1108633

Only save default metadata


thisisnic Apr 16, 2023
3e7e5fb

Implement collect.StructArray and update as.data.frame.StructArray to…


thisisnic Apr 17, 2023
cb3a0d2

as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…


thisisnic Apr 17, 2023
12eeff3

as.data.frame.arrow_dplyr_query returns data.frames


thisisnic Apr 17, 2023
85c6dc2

Pass through additional arguments


thisisnic Apr 17, 2023
42b02d1

Call collect not as.data.frame inside reader functions


thisisnic Apr 17, 2023
dd07cda

Implement helper function for comparing metadata


thisisnic Apr 17, 2023
46ad76a

Don't use collect.ArrowTabular


thisisnic Apr 21, 2023
8d7c78c

Update expect_data_frame to convert object and expected into data.fra…


thisisnic Apr 21, 2023
ab93ba5

Rename function for clarity


thisisnic Apr 21, 2023
33e7c6f

Update test-utf to use expect_equal_data_frame


thisisnic Apr 21, 2023
4e0562f

Only remove metadata if it is definitely default metadata


thisisnic Apr 21, 2023
652952c

Update schema test so it uses an object where even default metadata i…


thisisnic Apr 21, 2023
e52d3cf

Remove expect_metadata function


thisisnic Apr 24, 2023
a1ace5f

Update Python tests to compare dfs


thisisnic Apr 25, 2023
e9d21ee

Remove default metadata and nothing else


thisisnic Apr 25, 2023
5761ffa

Update code to only remove class


thisisnic Apr 25, 2023
090b5de

Relocate collect.StructArray


thisisnic Apr 26, 2023
a4cdba6

Merge branch 'apache:main' into GH-34775_new_as_df


thisisnic May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
r/R/array.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
 as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down

3 changes: 2 additions & 1 deletion
r/R/arrow-tabular.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 out <- apply_arrow_r_metadata(df, x$metadata$r)
 as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down

2 changes: 1 addition & 1 deletion
r/R/csv.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }

 tab
Expand Down

7 changes: 6 additions & 1 deletion
r/R/dplyr-collect.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
 if (as_data_frame) {
 as.data.frame(x, ...)
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 } else {
 x
 }
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
 # TODO: should this tryCatch move down into as_arrow_table()?
Expand Down

3 changes: 2 additions & 1 deletion
r/R/dplyr.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
 collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 as.data.frame(out)
}

#' @export
Expand Down

3 changes: 2 additions & 1 deletion
r/R/feather.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
 )

 if (isTRUE(as_data_frame)) {
 out <- as.data.frame(out)
 df <- out$to_data_frame()
 out <- apply_arrow_r_metadata(df, out$metadata$r)
 }
 out
}
Expand Down

2 changes: 1 addition & 1 deletion
r/R/ipc-stream.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
 # https://issues.apache.org/jira/browse/ARROW-6830
 out <- RecordBatchStreamReader$create(file)$read_table()
 if (as_data_frame) {
 out <- as.data.frame(out)
 out <- collect.ArrowTabular(out)
 }
 out
}

2 changes: 1 addition & 1 deletion
r/R/json.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

9 changes: 9 additions & 0 deletions
r/R/metadata.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
 # drop problems attributes (most likely from readr)
 x[["attributes"]][["problems"]] <- NULL

 # remove the class if it's just data.frame
 if (identical(x$attributes$class, "data.frame")) {
 x$attributes <- x$attributes[names(x$attributes) != "class"]
 if (is_empty(x$attributes)) {
 x <- x[names(x) != "attributes"]
 }
 }

 out <- serialize(x, NULL, ascii = TRUE)

 # if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
 expr = {
 columns_metadata <- r_metadata$columns
 if (is.data.frame(x)) {
 # if columns metadata exists, apply it here
 if (length(names(x)) && !is.null(columns_metadata)) {
 for (name in intersect(names(columns_metadata), names(x))) {
 x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down

2 changes: 1 addition & 1 deletion
r/R/parquet.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
 }

 if (as_data_frame) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

5 changes: 3 additions & 2 deletions
r/tests/testthat/helper-expectation.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
 expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down

83 changes: 41 additions & 42 deletions
r/tests/testthat/test-RecordBatch.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
 schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
 )
 expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

 # input validation
 expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
 batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

 batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

 # Input validation
 expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
 expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
 batch[rep(c(FALSE, TRUE), 5), ],
 tbl[c(2, 4, 6, 8, 10), ]
 )
 # bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
 # int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

 # input validation
 expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

 # can remove a column
 batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

 # can remove a column by index
 batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

 # can add a named column
 batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

 # can replace a column by index
 batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
 fct = factor(letters[1:10])
 )
 batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
 b = struct(x = int32(), y = int32())
 )
 )
 out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

 # if not named, columns from tib are auto spliced
 batch2 <- record_batch(a = 1:10, tib)
 expect_equal(
 batch2$schema,
 schema(a = int32(), x = int32(), y = int32())
 )
 out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
 schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
 batch <- record_batch(a = 1:10, b = tib, schema = schema)
 expect_equal(batch$schema, schema)
 out <- as.data.frame(batch)
 expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
 expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

 schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
 expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
 batch2 <- record_batch(!!!df)
 expect_equal(batch1, batch2)
 expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

 batch3 <- record_batch(df, z = 1:10)
 batch4 <- record_batch(!!!df, z = 1:10)
 expect_equal(batch3, batch4)
 expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch3,
 cbind(df, data.frame(z = 1:10))
 )

 s <- schema(x = float64(), y = utf8())
 batch5 <- record_batch(df, schema = s)
 batch6 <- record_batch(!!!df, schema = s)
 expect_equal(batch5, batch6)
 expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

 s2 <- schema(x = float64(), y = utf8(), z = int16())
 batch7 <- record_batch(df, z = 1:10, schema = s2)
 batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
 expect_equal(batch7, batch8)
 expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch7,
 cbind(df, data.frame(z = 1:10))
 )
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = 1:10, b = 5),
 tibble::tibble(a = 1:10, b = 5)
 )
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Scalar$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Array$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame by thisisnic · Pull Request #35173 · apache/arrow · GitHub
Skip to content
GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame#35173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

thisisnic
merged 19 commits into

from

May 3, 2023
 Merged


GH-34775: [R] arrow_table: as.data.frame() sometimes returns a tbl and sometimes a data.frame
#35173
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


19 commits


Select commit
Hold shift + click to select a range
1108633

Only save default metadata


thisisnic Apr 16, 2023
3e7e5fb

Implement collect.StructArray and update as.data.frame.StructArray to…


thisisnic Apr 17, 2023
cb3a0d2

as.data.frame.ArrowTabular returns data.frames and collect.ArrowTabul…


thisisnic Apr 17, 2023
12eeff3

as.data.frame.arrow_dplyr_query returns data.frames


thisisnic Apr 17, 2023
85c6dc2

Pass through additional arguments


thisisnic Apr 17, 2023
42b02d1

Call collect not as.data.frame inside reader functions


thisisnic Apr 17, 2023
dd07cda

Implement helper function for comparing metadata


thisisnic Apr 17, 2023
46ad76a

Don't use collect.ArrowTabular


thisisnic Apr 21, 2023
8d7c78c

Update expect_data_frame to convert object and expected into data.fra…


thisisnic Apr 21, 2023
ab93ba5

Rename function for clarity


thisisnic Apr 21, 2023
33e7c6f

Update test-utf to use expect_equal_data_frame


thisisnic Apr 21, 2023
4e0562f

Only remove metadata if it is definitely default metadata


thisisnic Apr 21, 2023
652952c

Update schema test so it uses an object where even default metadata i…


thisisnic Apr 21, 2023
e52d3cf

Remove expect_metadata function


thisisnic Apr 24, 2023
a1ace5f

Update Python tests to compare dfs


thisisnic Apr 25, 2023
e9d21ee

Remove default metadata and nothing else


thisisnic Apr 25, 2023
5761ffa

Update code to only remove class


thisisnic Apr 25, 2023
090b5de

Relocate collect.StructArray


thisisnic Apr 26, 2023
a4cdba6

Merge branch 'apache:main' into GH-34775_new_as_df


thisisnic May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
r/R/array.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -474,7 +474,7 @@ dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)

#' @export
as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
 as.data.frame(collect.StructArray(x), row.names = row.names, optional = optional, ...)
}

#' @rdname array
Expand Down

3 changes: 2 additions & 1 deletion
r/R/arrow-tabular.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -94,7 +94,8 @@ ArrowTabular <- R6Class("ArrowTabular",
#' @export
as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 out <- apply_arrow_r_metadata(df, x$metadata$r)
 as.data.frame(out, row.names = row.names, optional = optional, ...)
}

#' @export
Expand Down

2 changes: 1 addition & 1 deletion
r/R/csv.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -248,7 +248,7 @@ read_delim_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }

 tab
Expand Down

7 changes: 6 additions & 1 deletion
r/R/dplyr-collect.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,8 @@ collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
}
collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
 if (as_data_frame) {
 as.data.frame(x, ...)
 df <- x$to_data_frame()
 apply_arrow_r_metadata(df, x$metadata$r)
 } else {
 x
 }
Expand All@@ -34,6 +35,10 @@ collect.Dataset <- function(x, as_data_frame = TRUE, ...) {
}
collect.RecordBatchReader <- collect.Dataset

collect.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
 as.vector(x)
}

compute.ArrowTabular <- function(x, ...) x
compute.arrow_dplyr_query <- function(x, ...) {
 # TODO: should this tryCatch move down into as_arrow_table()?
Expand Down

3 changes: 2 additions & 1 deletion
r/R/dplyr.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -216,7 +216,8 @@ unique.RecordBatchReader <- unique.arrow_dplyr_query

#' @export
as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
 collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 out <- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
 as.data.frame(out)
}

#' @export
Expand Down

3 changes: 2 additions & 1 deletion
r/R/feather.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -196,7 +196,8 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T
 )

 if (isTRUE(as_data_frame)) {
 out <- as.data.frame(out)
 df <- out$to_data_frame()
 out <- apply_arrow_r_metadata(df, out$metadata$r)
 }
 out
}
Expand Down

2 changes: 1 addition & 1 deletion
r/R/ipc-stream.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
 # https://issues.apache.org/jira/browse/ARROW-6830
 out <- RecordBatchStreamReader$create(file)$read_table()
 if (as_data_frame) {
 out <- as.data.frame(out)
 out <- collect.ArrowTabular(out)
 }
 out
}

2 changes: 1 addition & 1 deletion
r/R/json.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ read_json_arrow <- function(file,
 }

 if (isTRUE(as_data_frame)) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

9 changes: 9 additions & 0 deletions
r/R/metadata.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,14 @@
 # drop problems attributes (most likely from readr)
 x[["attributes"]][["problems"]] <- NULL

 # remove the class if it's just data.frame
 if (identical(x$attributes$class, "data.frame")) {
 x$attributes <- x$attributes[names(x$attributes) != "class"]
 if (is_empty(x$attributes)) {
 x <- x[names(x) != "attributes"]
 }
 }

 out <- serialize(x, NULL, ascii = TRUE)

 # if the metadata is over 100 kB, compress
Expand DownExpand Up@@ -62,6 +70,7 @@ apply_arrow_r_metadata <- function(x, r_metadata) {
 expr = {
 columns_metadata <- r_metadata$columns
 if (is.data.frame(x)) {
 # if columns metadata exists, apply it here
 if (length(names(x)) && !is.null(columns_metadata)) {
 for (name in intersect(names(columns_metadata), names(x))) {
 x[[name]] <- apply_arrow_r_metadata(x[[name]], columns_metadata[[name]])
Expand Down

2 changes: 1 addition & 1 deletion
r/R/parquet.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -70,7 +70,7 @@ read_parquet <- function(file,
 }

 if (as_data_frame) {
 tab <- as.data.frame(tab)
 tab <- collect.ArrowTabular(tab)
 }
 tab
}
Expand Down

5 changes: 3 additions & 2 deletions
r/tests/testthat/helper-expectation.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,8 +19,9 @@ expect_as_vector <- function(x, y, ...) {
 expect_equal(as.vector(x), y, ...)
}

expect_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), y, ...)
# expect both objects to contain equal values when converted to data.frame objects
expect_equal_data_frame <- function(x, y, ...) {
 expect_equal(as.data.frame(x), as.data.frame(y), ...)
}

expect_r6_class <- function(object, class) {
Expand Down

83 changes: 41 additions & 42 deletions
r/tests/testthat/test-RecordBatch.R

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -89,7 +89,7 @@ test_that("RecordBatch", {
 schema(dbl = float64(), lgl = boolean(), chr = utf8(), fct = dictionary(int8(), utf8()))
 )
 expect_equal(batch2$column(0), batch$column(1))
expect_data_frame(batch2, tbl[, -1])
expect_equal_data_frame(batch2, tbl[, -1])

 # input validation
 expect_error(batch$RemoveColumn(NA), "'i' cannot be NA")
Expand All@@ -109,10 +109,10 @@ test_that("RecordBatch S3 methods", {

test_that("RecordBatch$Slice", {
 batch3 <- batch$Slice(5)
expect_data_frame(batch3, tbl[6:10, ])
expect_equal_data_frame(batch3, tbl[6:10, ])

 batch4 <- batch$Slice(5, 2)
expect_data_frame(batch4, tbl[6:7, ])
expect_equal_data_frame(batch4, tbl[6:7, ])

 # Input validation
 expect_error(batch$Slice("ten"))
Expand All@@ -131,20 +131,20 @@ test_that("RecordBatch$Slice", {
})

test_that("[ on RecordBatch", {
expect_data_frame(batch[6:7, ], tbl[6:7, ])
expect_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
expect_equal_data_frame(batch[6:7, ], tbl[6:7, ])
expect_equal_data_frame(batch[c(6, 7), ], tbl[6:7, ])
expect_equal_data_frame(batch[6:7, 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[, c("dbl", "fct")], tbl[, c(2, 5)])
 expect_identical(as.vector(batch[, "chr", drop = TRUE]), tbl$chr)
expect_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_data_frame(
expect_equal_data_frame(batch[c(7, 3, 5), 2:4], tbl[c(7, 3, 5), 2:4])
expect_equal_data_frame(
 batch[rep(c(FALSE, TRUE), 5), ],
 tbl[c(2, 4, 6, 8, 10), ]
 )
 # bool Array
expect_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
expect_equal_data_frame(batch[batch$lgl, ], tbl[tbl$lgl, ])
 # int Array
expect_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])
expect_equal_data_frame(batch[Array$create(5:6), 2:4], tbl[6:7, 2:4])

 # input validation
 expect_error(batch[, c("dbl", "NOTACOLUMN")], 'Column not found: "NOTACOLUMN"')
Expand DownExpand Up@@ -176,15 +176,15 @@ test_that("[[<- assignment", {

 # can remove a column
 batch[["chr"]] <- NULL
expect_data_frame(batch, tbl[-4])
expect_equal_data_frame(batch, tbl[-4])

 # can remove a column by index
 batch[[4]] <- NULL
expect_data_frame(batch, tbl[1:3])
expect_equal_data_frame(batch, tbl[1:3])

 # can add a named column
 batch[["new"]] <- letters[10:1]
expect_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))
expect_equal_data_frame(batch, dplyr::bind_cols(tbl[1:3], new = letters[10:1]))

 # can replace a column by index
 batch[[2]] <- as.numeric(10:1)
Expand DownExpand Up@@ -239,16 +239,16 @@ test_that("head and tail on RecordBatch", {
 fct = factor(letters[1:10])
 )
 batch <- RecordBatch$create(tbl)
expect_data_frame(head(batch), head(tbl))
expect_data_frame(head(batch, 4), head(tbl, 4))
expect_data_frame(head(batch, 40), head(tbl, 40))
expect_data_frame(head(batch, -4), head(tbl, -4))
expect_data_frame(head(batch, -40), head(tbl, -40))
expect_data_frame(tail(batch), tail(tbl))
expect_data_frame(tail(batch, 4), tail(tbl, 4))
expect_data_frame(tail(batch, 40), tail(tbl, 40))
expect_data_frame(tail(batch, -4), tail(tbl, -4))
expect_data_frame(tail(batch, -40), tail(tbl, -40))
expect_equal_data_frame(head(batch), head(tbl))
expect_equal_data_frame(head(batch, 4), head(tbl, 4))
expect_equal_data_frame(head(batch, 40), head(tbl, 40))
expect_equal_data_frame(head(batch, -4), head(tbl, -4))
expect_equal_data_frame(head(batch, -40), head(tbl, -40))
expect_equal_data_frame(tail(batch), tail(tbl))
expect_equal_data_frame(tail(batch, 4), tail(tbl, 4))
expect_equal_data_frame(tail(batch, 40), tail(tbl, 40))
expect_equal_data_frame(tail(batch, -4), tail(tbl, -4))
expect_equal_data_frame(tail(batch, -40), tail(tbl, -40))
})

test_that("RecordBatch print method", {
Expand DownExpand Up@@ -346,17 +346,17 @@ test_that("record_batch() handles data frame columns", {
 b = struct(x = int32(), y = int32())
 )
 )
 out <- as.data.frame(batch)
expect_equal(out, tibble::tibble(a = 1:10, b = tib))

expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib))

 # if not named, columns from tib are auto spliced
 batch2 <- record_batch(a = 1:10, tib)
 expect_equal(
 batch2$schema,
 schema(a = int32(), x = int32(), y = int32())
 )
 out <- as.data.frame(batch2)
expect_equal(out, tibble::tibble(a = 1:10, !!!tib))

expect_equal_data_frame(batch2, tibble::tibble(a = 1:10, !!!tib))
})

test_that("record_batch() handles data frame columns with schema spec", {
Expand All@@ -366,8 +366,7 @@ test_that("record_batch() handles data frame columns with schema spec", {
 schema <- schema(a = int32(), b = struct(x = int16(), y = float64()))
 batch <- record_batch(a = 1:10, b = tib, schema = schema)
 expect_equal(batch$schema, schema)
 out <- as.data.frame(batch)
 expect_equal(out, tibble::tibble(a = 1:10, b = tib_float))
 expect_equal_data_frame(batch, tibble::tibble(a = 1:10, b = tib_float))

 schema <- schema(a = int32(), b = struct(x = int16(), y = utf8()))
 expect_error(record_batch(a = 1:10, b = tib, schema = schema))
Expand All@@ -379,32 +378,32 @@ test_that("record_batch() auto splices (ARROW-5718)", {
 batch2 <- record_batch(!!!df)
 expect_equal(batch1, batch2)
 expect_equal(batch1$schema, schema(x = int32(), y = utf8()))
expect_data_frame(batch1, df)
expect_equal_data_frame(batch1, df)

 batch3 <- record_batch(df, z = 1:10)
 batch4 <- record_batch(!!!df, z = 1:10)
 expect_equal(batch3, batch4)
 expect_equal(batch3$schema, schema(x = int32(), y = utf8(), z = int32()))
expect_equal(
as.data.frame(batch3),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch3,
 cbind(df, data.frame(z = 1:10))
 )

 s <- schema(x = float64(), y = utf8())
 batch5 <- record_batch(df, schema = s)
 batch6 <- record_batch(!!!df, schema = s)
 expect_equal(batch5, batch6)
 expect_equal(batch5$schema, s)
expect_equal(as.data.frame(batch5), df)
expect_equal_data_frame(batch5, df)

 s2 <- schema(x = float64(), y = utf8(), z = int16())
 batch7 <- record_batch(df, z = 1:10, schema = s2)
 batch8 <- record_batch(!!!df, z = 1:10, schema = s2)
 expect_equal(batch7, batch8)
 expect_equal(batch7$schema, s2)
expect_equal(
as.data.frame(batch7),
tibble::as_tibble(cbind(df, data.frame(z = 1:10)))
expect_equal_data_frame(
 batch7,
 cbind(df, data.frame(z = 1:10))
 )
})

Expand All@@ -425,24 +424,24 @@ test_that("record_batch() handles null type (ARROW-7064)", {
})

test_that("record_batch() scalar recycling with vectors", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = 1:10, b = 5),
 tibble::tibble(a = 1:10, b = 5)
 )
})

test_that("record_batch() scalar recycling with Scalars, Arrays, and ChunkedArrays", {
expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Scalar$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = Array$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )

expect_data_frame(
expect_equal_data_frame(
 record_batch(a = Array$create(1:10), b = ChunkedArray$create(5)),
 tibble::tibble(a = 1:10, b = 5)
 )
Expand Down
Loading