Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9ed32b6
Make write_parquet() generic, internal impl using streams rather than…
romainfrancois Sep 20, 2019
aa34095
passing down the right stream
romainfrancois Sep 20, 2019
0e09ac8
lint
romainfrancois Sep 20, 2019
1b84ad4
Exposing classes parquet::arrow::ArrowWriterProperties and parquet::a…
romainfrancois Sep 23, 2019
09ea0ad
+ ParquetWriterProperties$create() and associated ParquetWriterProper…
romainfrancois Sep 24, 2019
fa8990b
Expose options from ParquetWriterProperties and ParquetArrowWriterPro…
romainfrancois Sep 24, 2019
b8337e1
lint
romainfrancois Sep 24, 2019
2dd2cb9
+ compression_level= in write_parquet()
romainfrancois Sep 25, 2019
1e3b5b6
document()
romainfrancois Sep 25, 2019
2f2ae00
More flexible compression= and compression_level=
romainfrancois Sep 25, 2019
4055f67
More flexible arguments use_dictionary= and write_statistics=
romainfrancois Sep 25, 2019
1166264
using make_valid_time_unit()
romainfrancois Sep 26, 2019
738ea6e
Remove $default() methods and use $create() wityh default arguments i…
romainfrancois Sep 26, 2019
72caaab
using assert_that()
romainfrancois Sep 26, 2019
7f1c184
align arguments following tidyverse style guide
romainfrancois Sep 26, 2019
d318a66
==.Table
romainfrancois Sep 26, 2019
6c4f003
add test helper so that we actually can test parquet roundtrip
romainfrancois Sep 26, 2019
86d9ff4
Remove the _ from builder classes
romainfrancois Sep 26, 2019
004cf90
M%ake compression_from_name() vectorized
romainfrancois Sep 26, 2019
9bee8de
define and use internal make_valid_version() function
romainfrancois Sep 26, 2019
1fdcc0b
suggestsions from @nealrichardson
romainfrancois Sep 26, 2019
00cc214
abstract various ParquetWriterPropertiesBuilder$set_*() methods
romainfrancois Sep 26, 2019
5ade52d
wrong length for use_dictionary and write_statistics
romainfrancois Sep 27, 2019
c5549de
Test ==.Table
romainfrancois Sep 27, 2019
66c51fd
added all.equal.Object() that uses ==
romainfrancois Sep 27, 2019
45ec63b
Update r/R/parquet.R
romainfrancois Sep 27, 2019
56dac33
Move read_parquet() and write_parquet() to top of the file
romainfrancois Sep 27, 2019
ecd9218
rework documentation for write_parquet()
romainfrancois Sep 27, 2019
9aff79b
implement ==.Object that calls $Equals instead of implementing for ea…
romainfrancois Sep 27, 2019
50555f8
rename arguments to `x` and `sink`
romainfrancois Sep 27, 2019
413dd41
test make_valid_version()
romainfrancois Sep 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions r/NAMESPACE
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,16 +3,12 @@
S3method("!=",Object)
S3method("$",RecordBatch)
S3method("$",Table)
S3method("==",Array)
S3method("==",DataType)
S3method("==",Field)
S3method("==",Message)
S3method("==",RecordBatch)
S3method("==",Schema)
S3method("==",Object)
S3method("[",RecordBatch)
S3method("[",Table)
S3method("[[",RecordBatch)
S3method("[[",Table)
S3method(all,equal.Object)
S3method(as.data.frame,RecordBatch)
S3method(as.data.frame,Table)
S3method(as.raw,Buffer)
Expand DownExpand Up@@ -84,6 +80,7 @@ export(MessageType)
export(MockOutputStream)
export(ParquetFileReader)
export(ParquetReaderProperties)
export(ParquetVersionType)
export(RandomAccessFile)
export(ReadableFile)
export(RecordBatchFileReader)
Expand DownExpand Up@@ -173,6 +170,7 @@ importFrom(rlang,dots_n)
importFrom(rlang,enquo)
importFrom(rlang,enquos)
importFrom(rlang,is_false)
importFrom(rlang,is_integerish)
importFrom(rlang,list2)
importFrom(rlang,quo_is_null)
importFrom(rlang,warn)
Expand Down
3 changes: 0 additions & 3 deletions r/R/array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,6 +156,3 @@ length.Array <- function(x) x$length()

#' @export
as.vector.Array <- function(x, mode) x$as_vector()

#' @export
`==.Array` <- function(x, y) x$Equals(y)
12 changes: 11 additions & 1 deletion r/R/arrow-package.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#' @importFrom R6 R6Class
#' @importFrom purrr map map_int map2
#' @importFrom assertthat assert_that
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos
#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish
#' @importFrom Rcpp sourceCpp
#' @importFrom tidyselect vars_select
#' @useDynLib arrow, .registration = TRUE
Expand DownExpand Up@@ -66,6 +66,16 @@ Object <- R6Class("Object",
#' @export
`!=.Object` <- function(lhs, rhs) !(lhs == rhs)

#' @export
`==.Object` <- function(x, y) {
x$Equals(y)
}

#' @export
all.equal.Object <- function(target, current, ...) {
target == current
}

shared_ptr <- function(class, xp) {
if (!shared_ptr_is_null(xp)) class$new(xp)
}
Expand Down
120 changes: 118 additions & 2 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion r/R/buffer.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,8 @@
Buffer <- R6Class("Buffer", inherit = Object,
public = list(
ZeroPadding = function() Buffer__ZeroPadding(self),
data = function() Buffer__data(self)
data = function() Buffer__data(self),
Equals = function(other) Buffer__Equals(self, other)
),

active = list(
Expand Down
3 changes: 3 additions & 0 deletions r/R/chunked-array.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,6 +83,9 @@ ChunkedArray <- R6Class("ChunkedArray", inherit = Object,
}
}
out
},
Equals = function(other) {
ChunkedArray__Equals(self, other)
}
),
active = list(
Expand Down
14 changes: 11 additions & 3 deletions r/R/compression.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,18 +36,26 @@
#' @rdname Codec
#' @name Codec
#' @export
Codec <- R6Class("Codec", inherit = Object)
Codec <- R6Class("Codec", inherit = Object,
active = list(
name = function() util___Codec__name(self),
level = function() abort("Codec$level() not yet implemented")
)
)
Codec$create <- function(type = "gzip", compression_level = NA) {
if (is.character(type)) {
type <- unique_ptr(Codec, util___Codec__Create(
CompressionType[[match.arg(toupper(type), names(CompressionType))]],
compression_level
compression_from_name(type), compression_level
))
}
assert_is(type, "Codec")
type
}

compression_from_name <- function(name) {
Comment thread
romainfrancois marked this conversation as resolved.
Outdated
map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
}

#' @title Compressed stream classes
#' @rdname compression
#' @name compression
Expand Down
6 changes: 6 additions & 0 deletions r/R/enums.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,3 +84,9 @@ CompressionType <- enum("Compression::type",
FileType <- enum("FileType",
NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L
)

#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
)
24 changes: 12 additions & 12 deletions r/R/feather.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,8 +17,8 @@

#' Write data in the Feather format
#'
#' @param data `data.frame` or RecordBatch
#' @param stream A file path or an OutputStream
#' @param x `data.frame` or RecordBatch
#' @param sink A file path or an OutputStream
#'
#' @export
#' @examples
Expand All@@ -30,20 +30,20 @@
#' })
#' }
#' @include arrow-package.R
write_feather <- function(data, stream) {
if (is.data.frame(data)) {
data <- record_batch(data)
write_feather <- function(x, sink) {
if (is.data.frame(x)) {
x <- record_batch(x)
}
assert_is(data, "RecordBatch")
assert_is(x, "RecordBatch")

if (is.character(stream)) {
stream <- FileOutputStream$create(stream)
on.exit(stream$close())
if (is.character(sink)) {
sink <- FileOutputStream$create(sink)
on.exit(sink$close())
}
assert_is(stream, "OutputStream")
assert_is(sink, "OutputStream")

writer <- FeatherTableWriter$create(stream)
ipc___TableWriter__RecordBatch__WriteFeather(writer, data)
writer <- FeatherTableWriter$create(sink)
ipc___TableWriter__RecordBatch__WriteFeather(writer, x)
}

#' @title FeatherTableWriter class
Expand Down
5 changes: 0 additions & 5 deletions r/R/field.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,11 +67,6 @@ Field$create <- function(name, type, metadata) {
shared_ptr(Field, Field__initialize(name, type, TRUE))
}

#' @export
`==.Field` <- function(lhs, rhs){
lhs$Equals(rhs)
}

#' @param name field name
#' @param type logical type, instance of [DataType]
#' @param metadata currently ignored
Expand Down
3 changes: 0 additions & 3 deletions r/R/message.R
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,9 +45,6 @@ Message <- R6Class("Message", inherit = Object,
)
)

#' @export
`==.Message` <- function(x, y) x$Equals(y)

#' @title class arrow::MessageReader
#'
#' @usage NULL
Expand Down
Loading