') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Unreasonable memory usage when using extract_array() on a sparse TileDBArray · Issue #35 · LTLA/TileDBArray · GitHub
Skip to content

Unreasonable memory usage when using extract_array() on a sparse TileDBArray #35

Open
Open
Unreasonable memory usage when using extract_array() on a sparse TileDBArray#35

Description

@hpages
hpages
opened on Apr 10, 2026

Let's first create a 28k x 15k sparse TileDBArray object:

library(HDF5Array)
library(ExperimentHub)
hub<- ExperimentHub()
brain_path<-hub[["EH1040"]] # 1.3 Million Brain Cell Datasetbrain<- HDF5Array(brain_path, "counts")
library(TileDBArray)
path<- tempfile()
## Takes about 30-40s (resulting dataset is 154M on disk):B<- writeTileDBArray(brain[ , 1:15000], sparse=TRUE, path=path)
dim(B)
# [1] 27998 15000

Extracting a random subset of 8000 x 5000 values uses about > 26 GB of memory on my laptop (Ubuntu Linux 24.04):

set.seed(111)
index<-list(sample(nrow(B), 8000), sample(ncol(B), 5000))
m<- extract_array(B, index) # 'top' command reports > 26 GB of memory usage

That's A LOT!

Trying to extract anything slightly bigger will exhaust the 32GB of RAM of my laptop and kill my R session (Linux OOM Killer in action).

For comparison, loading the full array in memory as an ordinary array (dense representation) only consumes 5.1 GB:

b<- as.array(B) # 'top' command reports about 5.1 GB of memory usage

And from there I can extract the random subset very efficiently:

m2<- extract_array(b, index)

but this of course kind of defeats the purpose of using a sparse representation in the first place.

H.

sessionInfo():

Rversion4.6.0 alpha (2026-04-05r89793)
Platform:x86_64-pc-linux-gnuRunningunder:Ubuntu24.04.4LTSMatrixproducts:defaultBLAS:/home/hpages/R/R-4.6.r89793/lib/libRblas.soLAPACK:/home/hpages/R/R-4.6.r89793/lib/libRlapack.so; LAPACKversion3.12.1locale:
[1] LC_CTYPE=en_US.UTF-8LC_NUMERIC=C [3] LC_TIME=en_GBLC_COLLATE=en_US.UTF-8 [5] LC_MONETARY=en_US.UTF-8LC_MESSAGES=en_US.UTF-8 [7] LC_PAPER=en_US.UTF-8LC_NAME=C [9] LC_ADDRESS=CLC_TELEPHONE=C [11] LC_MEASUREMENT=en_US.UTF-8LC_IDENTIFICATION=Ctimezone:America/Los_Angelestzcodesource: system (glibc)
attachedbasepackages:
[1] stats4statsgraphicsgrDevicesutilsdatasetsmethods [8] baseotherattachedpackages:
[1] RcppSpdlog_0.0.28TileDBArray_1.21.1 [3] TENxBrainData_1.31.0SingleCellExperiment_1.33.2
[5] SummarizedExperiment_1.41.1Biobase_2.71.0 [7] GenomicRanges_1.63.2Seqinfo_1.1.0 [9] ExperimentHub_3.1.0AnnotationHub_4.1.0 [11] BiocFileCache_3.1.0dbplyr_2.5.2 [13] HDF5Array_1.39.1h5mread_1.3.3 [15] rhdf5_2.55.16DelayedArray_0.37.1 [17] SparseArray_1.11.13S4Arrays_1.11.1 [19] IRanges_2.45.0abind_1.4-8 [21] S4Vectors_0.49.1MatrixGenerics_1.23.0 [23] matrixStats_1.5.0BiocGenerics_0.57.0 [25] generics_0.1.4Matrix_1.7-5loadedviaa namespace (andnotattached):
[1] KEGGREST_1.51.1httr2_1.2.2lattice_0.22-9 [4] rhdf5filters_1.23.3vctrs_0.7.2tools_4.6.0 [7] curl_7.0.0tibble_3.3.1AnnotationDbi_1.73.1
[10] RSQLite_2.4.6blob_1.3.0pkgconfig_2.0.3 [13] data.table_1.18.2.1lifecycle_1.0.5compiler_4.6.0 [16] Biostrings_2.79.5nanoarrow_0.8.0yaml_2.3.12 [19] pillar_1.11.1crayon_1.5.3cachem_1.1.0 [22] RcppCCTZ_0.2.14tiledb_0.33.0tidyselect_1.2.1 [25] dplyr_1.2.1purrr_1.2.1BiocVersion_3.23.1 [28] fastmap_1.2.0grid_4.6.0cli_3.6.6 [31] magrittr_2.0.5spdl_0.0.5withr_3.0.2 [34] filelock_1.0.3rappdirs_0.3.4bit64_4.6.0-1 [37] nanotime_0.3.13XVector_0.51.0httr_1.4.8 [40] bit_4.6.0zoo_1.8-15png_0.1-9 [43] memoise_2.0.1rlang_1.2.0Rcpp_1.1.1 [46] glue_1.8.0DBI_1.3.0BiocManager_1.30.27 [49] R6_2.6.1Rhdf5lib_1.33.6

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions