') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); ARROW-18113: [C++] Use RandomAccessFile::ReadManyAsync for cloud file systems by aucahuasi · Pull Request #14747 · apache/arrow · GitHub
Skip to content

ARROW-18113: [C++] Use RandomAccessFile::ReadManyAsync for cloud file systems - #14747

Closed
aucahuasi wants to merge 3 commits into
apache:mainfrom
aucahuasi:ARROW-18113-new-read-coalesce-cloudfs
Closed

ARROW-18113: [C++] Use RandomAccessFile::ReadManyAsync for cloud file systems#14747
aucahuasi wants to merge 3 commits into
apache:mainfrom
aucahuasi:ARROW-18113-new-read-coalesce-cloudfs

Conversation

@aucahuasi

@aucahuasiaucahuasi commented Nov 28, 2022•
edited
Loading

Copy link
Copy Markdown
Contributor

This PR is a follow up from #14723:

  • Rename arrow::io::CacheOptions to arrow::io::CoalesceOptions (deprecate CacheOptions)
  • Implement RandomAccessFile::ReadManyAsync for S3 file system.
  • Implement RandomAccessFile::ReadManyAsync for Google Cloud Storage file system.
  • Add tests.

Related Jira ticket:
https://issues.apache.org/jira/browse/ARROW-18113

@github-actions

github-actionsBot commented Nov 28, 2022

Copy link
Copy Markdown

https://issues.apache.org/jira/browse/ARROW-18113

lidavidm
lidavidm reviewed Nov 28, 2022
View reviewed changes
Comment threadcpp/src/arrow/filesystem/s3fs.h
std::shared_ptr<S3RetryStrategy> retry_strategy;

/// Coalesce options for requesting multiple reads at once
io::CoalesceOptions coalesceOptions = io::CoalesceOptions::Defaults();

@lidavidmlidavidmNov 28, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: follow the naming conventions (snake_case)

@aucahuasiaucahuasiNov 28, 2022

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, btw I'll use here io::CoalesceOptions::MakeFromNetworkMetrics(5, 500) as default value (i.e. hole_size_limit = 2.5 MiB and range_size_limit = 22.5 MiB)

Comment threadcpp/src/arrow/filesystem/s3fs.h
Comment on lines +174 to +175
int64_t hole_size_limit = 1;
int64_t range_size_limit = 100;

@lidavidmlidavidmNov 28, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need these separately?

@aucahuasiaucahuasiNov 28, 2022

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo, I forgot to delete those fields

Comment threadcpp/src/arrow/filesystem/s3fs.cc
Comment on lines +1135 to +1137
ctx, io::internal::CoalesceReadRanges(std::move(ranges),
coalesceOptions.hole_size_limit,
coalesceOptions.range_size_limit));

@lidavidmlidavidmNov 28, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since coalescing may merge ranges, you'll need to post-process things so that the returned futures map 1:1 with the original ranges. (May need to refactor things so you don't have to do the coalescing step twice?)

@aucahuasiaucahuasiNov 28, 2022

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, thanks David, I'm writing a test for verifying these cases.

@asfimportasfimport mentioned this pull request Nov 29, 2022
Closed
@github-actions

github-actionsBot commented Nov 18, 2025

Copy link
Copy Markdown

Thank you for your contribution. Unfortunately, this pull request has been marked as stale because it has had no activity in the past 365 days. Please remove the stale label or comment below, or this PR will be closed in 14 days. Feel free to re-open this if it has been closed in error. If you do not have repository permissions to reopen the PR, please tag a maintainer.

@github-actionsgithub-actionsBot added the Status: stale-warning Issues and PRs flagged as stale which are due to be closed if no indication otherwise label Nov 18, 2025
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@lidavidmlidavidmlidavidm left review comments
@pitroupitrouAwaiting requested review from pitroupitrou will be requested when the pull request is marked ready for reviewpitrou is a code owner

Assignees

No one assigned

Labels

Component: C++Status: stale-warningIssues and PRs flagged as stale which are due to be closed if no indication otherwise

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

2 participants

@aucahuasi@lidavidm