ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed

trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders

js/.vscode/launch.json

-Original file line number
+Diff line change
@@ Expand Up / @@ -37,8 +37,17 @@ @@
  // "TEST_MODULE": "umd"
  },
  "args": [
-// "-i",
+ "-i",
  "test/unit/",
+ // "test/unit/builders/",
+ // "test/unit/builders/builder-tests.ts",
+ // "test/unit/builders/int64-tests.ts",
+ // "test/unit/builders/uint64-tests.ts",
+ // "test/unit/builders/date-tests.ts",
+ // "test/unit/builders/primitive-tests.ts",
+ // "test/unit/builders/dictionary-tests.ts",
+ // "test/unit/builders/utf8-tests.ts",
  // Uncomment any of these to run individual test suites
  // "test/unit/int-tests.ts",
@@ Expand Down @@

js/bin/print-buffer-alignment.js

-Original file line number
+Diff line change
@@ Expand Up / @@ -22,29 +22,45 @@ @@
 const fs = require('fs');
 const path = require('path');
 const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '';
-const { AsyncMessageReader } = require(`../index${extension}`);
+const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`);
+const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`);
 (async () => {
  const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
  const reader = new AsyncMessageReader(readable);
- let recordBatchIndex = 0, dictionaryBatchIndex = 0;
+ let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0;
  for await (let message of reader) {
  let bufferRegions = [];
  if (message.isSchema()) {
+ schema = message.header();
  continue;
  } else if (message.isRecordBatch()) {
- bufferRegions = message.header().buffers;
+ const header = message.header();
+ bufferRegions = header.buffers;
  const body = await reader.readMessageBody(message.bodyLength);
- console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`);
+ const recordBatch = loadRecordBatch(schema, header, body);
+ console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({
+ offset: body.byteOffset,
+ length: body.byteLength,
+ numRows: recordBatch.length,
+ })}`);
  } else if (message.isDictionaryBatch()) {
- bufferRegions = message.header().data.buffers;
+ const header = message.header();
+ bufferRegions = header.data.buffers;
+ const type = schema.dictionaries.get(header.id);
  const body = await reader.readMessageBody(message.bodyLength);
- console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`);
+ const recordBatch = loadDictionaryBatch(header.data, body, type);
+ console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({
+ offset: body.byteOffset,
+ length: body.byteLength,
+ numRows: recordBatch.length,
+ dictionaryId: header.id,
+ })}`);
  }
  bufferRegions.forEach(({ offset, length }, i) => {
@@ Expand All @@
  await reader.return();
 })().catch((e) => { console.error(e); process.exit(1); });
+function loadRecordBatch(schema, header, body) {
+ return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
+}
+function loadDictionaryBatch(header, body, dictionaryType) {
+ return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
+}

js/gulp/closure-task.js

-Original file line number
+Diff line change
@@ Expand Up / @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ @@
  // formatting: `PRETTY_PRINT`,
  // debug: true,
  compilation_level: `ADVANCED`,
- allow_method_call_decomposing: true,
  package_json_entry_names: `module,jsnext:main,main`,
  assume_function_wrapper: true,
  js_output_file: `${mainExport}.js`,
@@ Expand Down @@

js/gulp/test-task.js

-Original file line number
+Diff line change
@@ Expand Up / @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); @@
 const jestArgv = [];
 argv.verbose && jestArgv.push(`--verbose`);
 argv.coverage
- ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`)
+ ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`)
  : jestArgv.push(`-c`, `jest.config.js`, `-i`)
 const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`);
@@ Expand Down @@

js/gulp/util.js

Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) {
 env: { ...process.env, NODE_NO_WARNINGS: `1` }
 };
 return asyncDone(() => child_process.spawn(`node`, args, opts))
 .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; });
 .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; });
}

const logAndDie = (e) => { if (e) { process.exit(1); } };
Expand Down

blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + ' Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /arrowPublic Notifications You must be signed in to change notification settings Fork 4.3k Star 17.1k Code Issues2.2k Pull requests401 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Closed trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders Conversation 23 Commits 56 Checks 0 Files changed Uh oh! There was an error while loading. Please reload this page. Closed ARROW-5115: [JS] Add Vector Builders and high-level stream primitives #4476 Show file tree Hide file tree Changes from all commits Commits Show all changes 56 commits Select commit Hold shift + click to select a range 738fabb WIP streaming data builders trxcllnt Feb 1, 2019ff32cbb fix date builder tests trxcllnt Feb 3, 20199e5e3ea fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196ebaa36 WIP streaming data builders trxcllnt Feb 1, 20192f423ec fix date builder tests trxcllnt Feb 3, 2019c609266 fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196b7a20e ensure 64-bit int builders use BN to check for nulls trxcllnt Feb 6, 20198a5c713 show a better error message if gulp fails trxcllnt Mar 20, 2019cf5f778 return signed or unsigned 53bit integers trxcllnt Mar 20, 20196e64c40 ensure builder allocates aligned buffers, null bitmaps initialized to… trxcllnt Mar 20, 20192b7b992 ensure union typeids buffer is an Int8Array trxcllnt Mar 20, 2019f7fe8c1 update builder buffer padding trxcllnt Apr 7, 20191398ebd don't clone Dictionary DataType instances in Schema assign to preserv… trxcllnt Apr 8, 2019fb2c9a2 cleanup trxcllnt Apr 9, 20192f36741 move stream methods to io folder trxcllnt Apr 15, 2019385fee3 fix typo trxcllnt Apr 15, 2019d193141 add helper method to return the stride for a datatype trxcllnt Apr 15, 201970377a1 add vectorname getter to chunked for completion trxcllnt Apr 15, 2019f8760cb enumerate each type key individually now that they're a real thing trxcllnt Apr 15, 20199ba9965 update row type inference sanity check trxcllnt Apr 15, 20194c2ef42 add the rest of the builder types trxcllnt Apr 15, 2019a307273 fix readable-stream detection in firefox trxcllnt Apr 16, 2019f651657 update typescript, ts-jest, jest trxcllnt Apr 16, 2019f708190 use safe BigInt64Array constructor trxcllnt Apr 17, 201915ac17c use Object.setPrototypeOf to improve bn performance trxcllnt May 11, 20190990344 add Builder throughDOM and throughNode transform streams trxcllnt May 17, 20192b7a63c update test types trxcllnt May 22, 201979bfd46 fix node and dom builder streams trxcllnt May 22, 201926387c7 update typescript, finish streaming builders, add comprehensive build… trxcllnt May 27, 2019fd46163 fix nan checks trxcllnt May 27, 2019d032e2d update dependencies trxcllnt May 27, 201986a6cd0 update closure compiler dependency trxcllnt May 27, 2019c587c33 add BufferBuilders, clean up public Builder API trxcllnt May 29, 20190209e9d update to typescript 3.5 trxcllnt May 30, 201984beb68 remove some property getters, clean up trxcllnt May 30, 2019dddba1a ensure variablewidthbuilder supports random insertion trxcllnt May 31, 2019893c74f ensure ListBuilder supports random insertion trxcllnt Jun 1, 2019c2673ac add initial builder jsdoc trxcllnt Jun 4, 20195b990c3 Clean up typedoc output, update typedoc to master to use typescript@3… trxcllnt Jun 5, 2019b2556b7 add docstring for _InternalEmptyPlaceholderRecordBatch trxcllnt Jun 5, 20194b5375e remove unnecessary jsdoc trxcllnt Jun 5, 20195d4b0be ensure bitmapbufferbuilder increments and decrements _popCount approp… trxcllnt Jun 5, 2019e974444 fix lint trxcllnt Jun 5, 20195aea4f9 Add more specific Int64 and Uint64 Builder tests trxcllnt Jun 5, 20193adf555 adds or updates most of the high-level Vector.from() methods to use t… trxcllnt Jun 6, 2019f250845 use a better default dictionary builder hash function trxcllnt Jun 6, 201913b51db print more details about each message trxcllnt Jun 6, 20198659323 add test for builder iterable byte queueing strategy trxcllnt Jun 6, 2019ba755ad use 53 bit hash fn to further avoid collisions trxcllnt Jun 6, 20199c3a865 ensure byteLength is calculated for offsets buffer trxcllnt Jun 6, 2019c0f7f7b fix a few minor formatting issues in arrow2csv trxcllnt Jun 7, 201905acad8 fix minor row serialization issues to be compatible with console.table() trxcllnt Jun 7, 20194d2a4f0 regenerate flatbuffer source from current format schemas trxcllnt Jun 7, 20198b0752f fix possible AsyncRandomAccessFile race condition retrieving filehand… trxcllnt Jun 7, 201907fa443 remove default dictionary hash function trxcllnt Jun 19, 20197998d2a add createIsValidFunction example docstring trxcllnt Jun 19, 2019 File filter Filter by extension Filter by extension .js (5) .json (4) .ts (117) All 3 file types selected Only manifest files Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace js/.vscode/launch.json launch.json js/bin/print-buffer-alignment.js print-buffer-alignment.js js/gulp/closure-task.js closure-task.js js/gulp/test-task.js test-task.js js/gulp/util.js util.js js/package-lock.json package-lock.json js/package.json package.json js/src/Arrow.dom.ts Arrow.dom.ts js/src/Arrow.node.ts Arrow.node.ts js/src/Arrow.ts Arrow.ts js/src/bin/arrow2csv.ts arrow2csv.ts js/src/builder.ts builder.ts js/src/builder/binary.ts binary.ts js/src/builder/bool.ts bool.ts js/src/builder/buffer.ts buffer.ts js/src/builder/date.ts date.ts js/src/builder/decimal.ts decimal.ts js/src/builder/dictionary.ts dictionary.ts js/src/builder/fixedsizebinary.ts fixedsizebinary.ts js/src/builder/fixedsizelist.ts fixedsizelist.ts js/src/builder/float.ts float.ts js/src/builder/index.ts index.ts js/src/builder/int.ts int.ts js/src/builder/interval.ts interval.ts js/src/builder/list.ts list.ts js/src/builder/map.ts map.ts js/src/builder/null.ts null.ts js/src/builder/run.ts run.ts js/src/builder/struct.ts struct.ts js/src/builder/time.ts time.ts js/src/builder/timestamp.ts timestamp.ts js/src/builder/union.ts union.ts js/src/builder/utf8.ts utf8.ts js/src/builder/valid.ts valid.ts js/src/column.ts column.ts js/src/compute/dataframe.ts dataframe.ts js/src/data.ts data.ts js/src/enum.ts enum.ts js/src/fb/File.ts File.ts js/src/fb/Message.ts Message.ts js/src/fb/Schema.ts Schema.ts js/src/interfaces.ts interfaces.ts js/src/io/adapters.ts adapters.ts js/src/io/file.ts file.ts js/src/io/node/builder.ts builder.ts js/src/io/node/iterable.ts iterable.ts js/src/io/node/reader.ts reader.ts js/src/io/node/writer.ts writer.ts js/src/io/stream.ts stream.ts js/src/io/whatwg/builder.ts builder.ts js/src/io/whatwg/iterable.ts iterable.ts js/src/io/whatwg/reader.ts reader.ts js/src/io/whatwg/writer.ts writer.ts js/src/ipc/metadata/json.ts json.ts js/src/ipc/metadata/message.ts message.ts js/src/ipc/writer.ts writer.ts js/src/recordbatch.ts recordbatch.ts js/src/schema.ts schema.ts js/src/table.ts table.ts js/src/type.ts type.ts js/src/util/args.ts args.ts js/src/util/bn.ts bn.ts js/src/util/buffer.ts buffer.ts js/src/util/compat.ts compat.ts js/src/util/fn.ts fn.ts js/src/util/int.ts int.ts js/src/util/utf8.ts utf8.ts js/src/util/vector.ts vector.ts js/src/vector.ts vector.ts js/src/vector/base.ts base.ts js/src/vector/binary.ts binary.ts js/src/vector/bool.ts bool.ts js/src/vector/chunked.ts chunked.ts js/src/vector/date.ts date.ts js/src/vector/decimal.ts decimal.ts js/src/vector/dictionary.ts dictionary.ts js/src/vector/fixedsizebinary.ts fixedsizebinary.ts js/src/vector/fixedsizelist.ts fixedsizelist.ts js/src/vector/float.ts float.ts js/src/vector/index.ts index.ts js/src/vector/int.ts int.ts js/src/vector/interval.ts interval.ts js/src/vector/list.ts list.ts js/src/vector/map.ts map.ts js/src/vector/null.ts null.ts js/src/vector/row.ts row.ts js/src/vector/struct.ts struct.ts js/src/vector/time.ts time.ts js/src/vector/timestamp.ts timestamp.ts js/src/vector/union.ts union.ts js/src/vector/utf8.ts utf8.ts js/src/visitor/builderctor.ts builderctor.ts js/src/visitor/bytewidth.ts bytewidth.ts js/src/visitor/get.ts get.ts js/src/visitor/indexof.ts indexof.ts js/src/visitor/iterator.ts iterator.ts js/src/visitor/jsontypeassembler.ts jsontypeassembler.ts js/src/visitor/jsonvectorassembler.ts jsonvectorassembler.ts js/src/visitor/set.ts set.ts js/src/visitor/toarray.ts toarray.ts js/src/visitor/typeassembler.ts typeassembler.ts js/src/visitor/typecomparator.ts typecomparator.ts js/src/visitor/typector.ts typector.ts js/src/visitor/vectorassembler.ts vectorassembler.ts js/src/visitor/vectorctor.ts vectorctor.ts js/src/visitor/vectorloader.ts vectorloader.ts js/test/Arrow.ts Arrow.ts js/test/generate-test-data.ts generate-test-data.ts js/test/inference/column.ts column.ts js/test/inference/nested.ts nested.ts js/test/jest-extensions.ts jest-extensions.ts js/test/unit/builders/builder-tests.ts builder-tests.ts js/test/unit/builders/date-tests.ts date-tests.ts js/test/unit/builders/dictionary-tests.ts dictionary-tests.ts js/test/unit/builders/int64-tests.ts int64-tests.ts js/test/unit/builders/primitive-tests.ts primitive-tests.ts js/test/unit/builders/uint64-tests.ts uint64-tests.ts js/test/unit/builders/utf8-tests.ts utf8-tests.ts js/test/unit/builders/utils.ts utils.ts js/test/unit/generated-data-tests.ts generated-data-tests.ts js/test/unit/ipc/writer/streams-dom-tests.ts streams-dom-tests.ts js/test/unit/ipc/writer/streams-node-tests.ts streams-node-tests.ts js/test/unit/vector/bool-vector-tests.ts bool-vector-tests.ts js/test/unit/vector/numeric-vector-tests.ts numeric-vector-tests.ts js/tsconfig/tsconfig.base.json tsconfig.base.json js/typedoc.js typedoc.js There are no files selected for viewing 11 changes: 10 additions & 1 deletion 11 js/.vscode/launch.json Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -37,8 +37,17 @@ // "TEST_MODULE": "umd" }, "args": [ // "-i", "-i", "test/unit/", // "test/unit/builders/", // "test/unit/builders/builder-tests.ts", // "test/unit/builders/int64-tests.ts", // "test/unit/builders/uint64-tests.ts", // "test/unit/builders/date-tests.ts", // "test/unit/builders/primitive-tests.ts", // "test/unit/builders/dictionary-tests.ts", // "test/unit/builders/utf8-tests.ts", // Uncomment any of these to run individual test suites // "test/unit/int-tests.ts", Expand Down 36 changes: 30 additions & 6 deletions 36 js/bin/print-buffer-alignment.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,29 +22,45 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; const { AsyncMessageReader } = require(`../index${extension}`); const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); let recordBatchIndex = 0, dictionaryBatchIndex = 0; let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; for await (let message of reader) { let bufferRegions = []; if (message.isSchema()) { schema = message.header(); continue; } else if (message.isRecordBatch()) { bufferRegions = message.header().buffers; const header = message.header(); bufferRegions = header.buffers; const body = await reader.readMessageBody(message.bodyLength); console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadRecordBatch(schema, header, body); console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, })}`); } else if (message.isDictionaryBatch()) { bufferRegions = message.header().data.buffers; const header = message.header(); bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); const body = await reader.readMessageBody(message.bodyLength); console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadDictionaryBatch(header.data, body, type); console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, dictionaryId: header.id, })}`); } bufferRegions.forEach(({ offset, length }, i) => { Expand All @@ -55,3 +71,11 @@ const { AsyncMessageReader } = require(`../index${extension}`); await reader.return(); })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields)); } function loadDictionaryBatch(header, body, dictionaryType) { return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType])); } 1 change: 0 additions & 1 deletion 1 js/gulp/closure-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ // formatting: `PRETTY_PRINT`, // debug: true, compilation_level: `ADVANCED`, allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/test-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.verbose && jestArgv.push(`--verbose`); argv.coverage ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`) ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`) : jestArgv.push(`-c`, `jest.config.js`, `-i`) const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/util.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) { env: { ...process.env, NODE_NO_WARNINGS: `1` } }; return asyncDone(() => child_process.spawn(`node`, args, opts)) .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; }); .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); } const logAndDie = (e) => { if (e) { process.exit(1); } }; Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /arrowPublic Notifications You must be signed in to change notification settings Fork 4.3k Star 17.1k Code Issues2.2k Pull requests401 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Closed trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders Conversation 23 Commits 56 Checks 0 Files changed Uh oh! There was an error while loading. Please reload this page. Closed ARROW-5115: [JS] Add Vector Builders and high-level stream primitives #4476 Show file tree Hide file tree Changes from all commits Commits Show all changes 56 commits Select commit Hold shift + click to select a range 738fabb WIP streaming data builders trxcllnt Feb 1, 2019ff32cbb fix date builder tests trxcllnt Feb 3, 20199e5e3ea fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196ebaa36 WIP streaming data builders trxcllnt Feb 1, 20192f423ec fix date builder tests trxcllnt Feb 3, 2019c609266 fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196b7a20e ensure 64-bit int builders use BN to check for nulls trxcllnt Feb 6, 20198a5c713 show a better error message if gulp fails trxcllnt Mar 20, 2019cf5f778 return signed or unsigned 53bit integers trxcllnt Mar 20, 20196e64c40 ensure builder allocates aligned buffers, null bitmaps initialized to… trxcllnt Mar 20, 20192b7b992 ensure union typeids buffer is an Int8Array trxcllnt Mar 20, 2019f7fe8c1 update builder buffer padding trxcllnt Apr 7, 20191398ebd don't clone Dictionary DataType instances in Schema assign to preserv… trxcllnt Apr 8, 2019fb2c9a2 cleanup trxcllnt Apr 9, 20192f36741 move stream methods to io folder trxcllnt Apr 15, 2019385fee3 fix typo trxcllnt Apr 15, 2019d193141 add helper method to return the stride for a datatype trxcllnt Apr 15, 201970377a1 add vectorname getter to chunked for completion trxcllnt Apr 15, 2019f8760cb enumerate each type key individually now that they're a real thing trxcllnt Apr 15, 20199ba9965 update row type inference sanity check trxcllnt Apr 15, 20194c2ef42 add the rest of the builder types trxcllnt Apr 15, 2019a307273 fix readable-stream detection in firefox trxcllnt Apr 16, 2019f651657 update typescript, ts-jest, jest trxcllnt Apr 16, 2019f708190 use safe BigInt64Array constructor trxcllnt Apr 17, 201915ac17c use Object.setPrototypeOf to improve bn performance trxcllnt May 11, 20190990344 add Builder throughDOM and throughNode transform streams trxcllnt May 17, 20192b7a63c update test types trxcllnt May 22, 201979bfd46 fix node and dom builder streams trxcllnt May 22, 201926387c7 update typescript, finish streaming builders, add comprehensive build… trxcllnt May 27, 2019fd46163 fix nan checks trxcllnt May 27, 2019d032e2d update dependencies trxcllnt May 27, 201986a6cd0 update closure compiler dependency trxcllnt May 27, 2019c587c33 add BufferBuilders, clean up public Builder API trxcllnt May 29, 20190209e9d update to typescript 3.5 trxcllnt May 30, 201984beb68 remove some property getters, clean up trxcllnt May 30, 2019dddba1a ensure variablewidthbuilder supports random insertion trxcllnt May 31, 2019893c74f ensure ListBuilder supports random insertion trxcllnt Jun 1, 2019c2673ac add initial builder jsdoc trxcllnt Jun 4, 20195b990c3 Clean up typedoc output, update typedoc to master to use typescript@3… trxcllnt Jun 5, 2019b2556b7 add docstring for _InternalEmptyPlaceholderRecordBatch trxcllnt Jun 5, 20194b5375e remove unnecessary jsdoc trxcllnt Jun 5, 20195d4b0be ensure bitmapbufferbuilder increments and decrements _popCount approp… trxcllnt Jun 5, 2019e974444 fix lint trxcllnt Jun 5, 20195aea4f9 Add more specific Int64 and Uint64 Builder tests trxcllnt Jun 5, 20193adf555 adds or updates most of the high-level Vector.from() methods to use t… trxcllnt Jun 6, 2019f250845 use a better default dictionary builder hash function trxcllnt Jun 6, 201913b51db print more details about each message trxcllnt Jun 6, 20198659323 add test for builder iterable byte queueing strategy trxcllnt Jun 6, 2019ba755ad use 53 bit hash fn to further avoid collisions trxcllnt Jun 6, 20199c3a865 ensure byteLength is calculated for offsets buffer trxcllnt Jun 6, 2019c0f7f7b fix a few minor formatting issues in arrow2csv trxcllnt Jun 7, 201905acad8 fix minor row serialization issues to be compatible with console.table() trxcllnt Jun 7, 20194d2a4f0 regenerate flatbuffer source from current format schemas trxcllnt Jun 7, 20198b0752f fix possible AsyncRandomAccessFile race condition retrieving filehand… trxcllnt Jun 7, 201907fa443 remove default dictionary hash function trxcllnt Jun 19, 20197998d2a add createIsValidFunction example docstring trxcllnt Jun 19, 2019 File filter Filter by extension Filter by extension .js (5) .json (4) .ts (117) All 3 file types selected Only manifest files Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace js/.vscode/launch.json launch.json js/bin/print-buffer-alignment.js print-buffer-alignment.js js/gulp/closure-task.js closure-task.js js/gulp/test-task.js test-task.js js/gulp/util.js util.js js/package-lock.json package-lock.json js/package.json package.json js/src/Arrow.dom.ts Arrow.dom.ts js/src/Arrow.node.ts Arrow.node.ts js/src/Arrow.ts Arrow.ts js/src/bin/arrow2csv.ts arrow2csv.ts js/src/builder.ts builder.ts js/src/builder/binary.ts binary.ts js/src/builder/bool.ts bool.ts js/src/builder/buffer.ts buffer.ts js/src/builder/date.ts date.ts js/src/builder/decimal.ts decimal.ts js/src/builder/dictionary.ts dictionary.ts js/src/builder/fixedsizebinary.ts fixedsizebinary.ts js/src/builder/fixedsizelist.ts fixedsizelist.ts js/src/builder/float.ts float.ts js/src/builder/index.ts index.ts js/src/builder/int.ts int.ts js/src/builder/interval.ts interval.ts js/src/builder/list.ts list.ts js/src/builder/map.ts map.ts js/src/builder/null.ts null.ts js/src/builder/run.ts run.ts js/src/builder/struct.ts struct.ts js/src/builder/time.ts time.ts js/src/builder/timestamp.ts timestamp.ts js/src/builder/union.ts union.ts js/src/builder/utf8.ts utf8.ts js/src/builder/valid.ts valid.ts js/src/column.ts column.ts js/src/compute/dataframe.ts dataframe.ts js/src/data.ts data.ts js/src/enum.ts enum.ts js/src/fb/File.ts File.ts js/src/fb/Message.ts Message.ts js/src/fb/Schema.ts Schema.ts js/src/interfaces.ts interfaces.ts js/src/io/adapters.ts adapters.ts js/src/io/file.ts file.ts js/src/io/node/builder.ts builder.ts js/src/io/node/iterable.ts iterable.ts js/src/io/node/reader.ts reader.ts js/src/io/node/writer.ts writer.ts js/src/io/stream.ts stream.ts js/src/io/whatwg/builder.ts builder.ts js/src/io/whatwg/iterable.ts iterable.ts js/src/io/whatwg/reader.ts reader.ts js/src/io/whatwg/writer.ts writer.ts js/src/ipc/metadata/json.ts json.ts js/src/ipc/metadata/message.ts message.ts js/src/ipc/writer.ts writer.ts js/src/recordbatch.ts recordbatch.ts js/src/schema.ts schema.ts js/src/table.ts table.ts js/src/type.ts type.ts js/src/util/args.ts args.ts js/src/util/bn.ts bn.ts js/src/util/buffer.ts buffer.ts js/src/util/compat.ts compat.ts js/src/util/fn.ts fn.ts js/src/util/int.ts int.ts js/src/util/utf8.ts utf8.ts js/src/util/vector.ts vector.ts js/src/vector.ts vector.ts js/src/vector/base.ts base.ts js/src/vector/binary.ts binary.ts js/src/vector/bool.ts bool.ts js/src/vector/chunked.ts chunked.ts js/src/vector/date.ts date.ts js/src/vector/decimal.ts decimal.ts js/src/vector/dictionary.ts dictionary.ts js/src/vector/fixedsizebinary.ts fixedsizebinary.ts js/src/vector/fixedsizelist.ts fixedsizelist.ts js/src/vector/float.ts float.ts js/src/vector/index.ts index.ts js/src/vector/int.ts int.ts js/src/vector/interval.ts interval.ts js/src/vector/list.ts list.ts js/src/vector/map.ts map.ts js/src/vector/null.ts null.ts js/src/vector/row.ts row.ts js/src/vector/struct.ts struct.ts js/src/vector/time.ts time.ts js/src/vector/timestamp.ts timestamp.ts js/src/vector/union.ts union.ts js/src/vector/utf8.ts utf8.ts js/src/visitor/builderctor.ts builderctor.ts js/src/visitor/bytewidth.ts bytewidth.ts js/src/visitor/get.ts get.ts js/src/visitor/indexof.ts indexof.ts js/src/visitor/iterator.ts iterator.ts js/src/visitor/jsontypeassembler.ts jsontypeassembler.ts js/src/visitor/jsonvectorassembler.ts jsonvectorassembler.ts js/src/visitor/set.ts set.ts js/src/visitor/toarray.ts toarray.ts js/src/visitor/typeassembler.ts typeassembler.ts js/src/visitor/typecomparator.ts typecomparator.ts js/src/visitor/typector.ts typector.ts js/src/visitor/vectorassembler.ts vectorassembler.ts js/src/visitor/vectorctor.ts vectorctor.ts js/src/visitor/vectorloader.ts vectorloader.ts js/test/Arrow.ts Arrow.ts js/test/generate-test-data.ts generate-test-data.ts js/test/inference/column.ts column.ts js/test/inference/nested.ts nested.ts js/test/jest-extensions.ts jest-extensions.ts js/test/unit/builders/builder-tests.ts builder-tests.ts js/test/unit/builders/date-tests.ts date-tests.ts js/test/unit/builders/dictionary-tests.ts dictionary-tests.ts js/test/unit/builders/int64-tests.ts int64-tests.ts js/test/unit/builders/primitive-tests.ts primitive-tests.ts js/test/unit/builders/uint64-tests.ts uint64-tests.ts js/test/unit/builders/utf8-tests.ts utf8-tests.ts js/test/unit/builders/utils.ts utils.ts js/test/unit/generated-data-tests.ts generated-data-tests.ts js/test/unit/ipc/writer/streams-dom-tests.ts streams-dom-tests.ts js/test/unit/ipc/writer/streams-node-tests.ts streams-node-tests.ts js/test/unit/vector/bool-vector-tests.ts bool-vector-tests.ts js/test/unit/vector/numeric-vector-tests.ts numeric-vector-tests.ts js/tsconfig/tsconfig.base.json tsconfig.base.json js/typedoc.js typedoc.js There are no files selected for viewing 11 changes: 10 additions & 1 deletion 11 js/.vscode/launch.json Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -37,8 +37,17 @@ // "TEST_MODULE": "umd" }, "args": [ // "-i", "-i", "test/unit/", // "test/unit/builders/", // "test/unit/builders/builder-tests.ts", // "test/unit/builders/int64-tests.ts", // "test/unit/builders/uint64-tests.ts", // "test/unit/builders/date-tests.ts", // "test/unit/builders/primitive-tests.ts", // "test/unit/builders/dictionary-tests.ts", // "test/unit/builders/utf8-tests.ts", // Uncomment any of these to run individual test suites // "test/unit/int-tests.ts", Expand Down 36 changes: 30 additions & 6 deletions 36 js/bin/print-buffer-alignment.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,29 +22,45 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; const { AsyncMessageReader } = require(`../index${extension}`); const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); let recordBatchIndex = 0, dictionaryBatchIndex = 0; let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; for await (let message of reader) { let bufferRegions = []; if (message.isSchema()) { schema = message.header(); continue; } else if (message.isRecordBatch()) { bufferRegions = message.header().buffers; const header = message.header(); bufferRegions = header.buffers; const body = await reader.readMessageBody(message.bodyLength); console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadRecordBatch(schema, header, body); console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, })}`); } else if (message.isDictionaryBatch()) { bufferRegions = message.header().data.buffers; const header = message.header(); bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); const body = await reader.readMessageBody(message.bodyLength); console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadDictionaryBatch(header.data, body, type); console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, dictionaryId: header.id, })}`); } bufferRegions.forEach(({ offset, length }, i) => { Expand All @@ -55,3 +71,11 @@ const { AsyncMessageReader } = require(`../index${extension}`); await reader.return(); })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields)); } function loadDictionaryBatch(header, body, dictionaryType) { return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType])); } 1 change: 0 additions & 1 deletion 1 js/gulp/closure-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ // formatting: `PRETTY_PRINT`, // debug: true, compilation_level: `ADVANCED`, allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/test-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.verbose && jestArgv.push(`--verbose`); argv.coverage ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`) ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`) : jestArgv.push(`-c`, `jest.config.js`, `-i`) const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/util.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) { env: { ...process.env, NODE_NO_WARNINGS: `1` } }; return asyncDone(() => child_process.spawn(`node`, args, opts)) .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; }); .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); } const logAndDie = (e) => { if (e) { process.exit(1); } }; Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /arrowPublic Notifications You must be signed in to change notification settings Fork 4.3k Star 17.1k Code Issues2.2k Pull requests401 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Closed trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders Conversation 23 Commits 56 Checks 0 Files changed Uh oh! There was an error while loading. Please reload this page. Closed ARROW-5115: [JS] Add Vector Builders and high-level stream primitives #4476 Show file tree Hide file tree Changes from all commits Commits Show all changes 56 commits Select commit Hold shift + click to select a range 738fabb WIP streaming data builders trxcllnt Feb 1, 2019ff32cbb fix date builder tests trxcllnt Feb 3, 20199e5e3ea fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196ebaa36 WIP streaming data builders trxcllnt Feb 1, 20192f423ec fix date builder tests trxcllnt Feb 3, 2019c609266 fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196b7a20e ensure 64-bit int builders use BN to check for nulls trxcllnt Feb 6, 20198a5c713 show a better error message if gulp fails trxcllnt Mar 20, 2019cf5f778 return signed or unsigned 53bit integers trxcllnt Mar 20, 20196e64c40 ensure builder allocates aligned buffers, null bitmaps initialized to… trxcllnt Mar 20, 20192b7b992 ensure union typeids buffer is an Int8Array trxcllnt Mar 20, 2019f7fe8c1 update builder buffer padding trxcllnt Apr 7, 20191398ebd don't clone Dictionary DataType instances in Schema assign to preserv… trxcllnt Apr 8, 2019fb2c9a2 cleanup trxcllnt Apr 9, 20192f36741 move stream methods to io folder trxcllnt Apr 15, 2019385fee3 fix typo trxcllnt Apr 15, 2019d193141 add helper method to return the stride for a datatype trxcllnt Apr 15, 201970377a1 add vectorname getter to chunked for completion trxcllnt Apr 15, 2019f8760cb enumerate each type key individually now that they're a real thing trxcllnt Apr 15, 20199ba9965 update row type inference sanity check trxcllnt Apr 15, 20194c2ef42 add the rest of the builder types trxcllnt Apr 15, 2019a307273 fix readable-stream detection in firefox trxcllnt Apr 16, 2019f651657 update typescript, ts-jest, jest trxcllnt Apr 16, 2019f708190 use safe BigInt64Array constructor trxcllnt Apr 17, 201915ac17c use Object.setPrototypeOf to improve bn performance trxcllnt May 11, 20190990344 add Builder throughDOM and throughNode transform streams trxcllnt May 17, 20192b7a63c update test types trxcllnt May 22, 201979bfd46 fix node and dom builder streams trxcllnt May 22, 201926387c7 update typescript, finish streaming builders, add comprehensive build… trxcllnt May 27, 2019fd46163 fix nan checks trxcllnt May 27, 2019d032e2d update dependencies trxcllnt May 27, 201986a6cd0 update closure compiler dependency trxcllnt May 27, 2019c587c33 add BufferBuilders, clean up public Builder API trxcllnt May 29, 20190209e9d update to typescript 3.5 trxcllnt May 30, 201984beb68 remove some property getters, clean up trxcllnt May 30, 2019dddba1a ensure variablewidthbuilder supports random insertion trxcllnt May 31, 2019893c74f ensure ListBuilder supports random insertion trxcllnt Jun 1, 2019c2673ac add initial builder jsdoc trxcllnt Jun 4, 20195b990c3 Clean up typedoc output, update typedoc to master to use typescript@3… trxcllnt Jun 5, 2019b2556b7 add docstring for _InternalEmptyPlaceholderRecordBatch trxcllnt Jun 5, 20194b5375e remove unnecessary jsdoc trxcllnt Jun 5, 20195d4b0be ensure bitmapbufferbuilder increments and decrements _popCount approp… trxcllnt Jun 5, 2019e974444 fix lint trxcllnt Jun 5, 20195aea4f9 Add more specific Int64 and Uint64 Builder tests trxcllnt Jun 5, 20193adf555 adds or updates most of the high-level Vector.from() methods to use t… trxcllnt Jun 6, 2019f250845 use a better default dictionary builder hash function trxcllnt Jun 6, 201913b51db print more details about each message trxcllnt Jun 6, 20198659323 add test for builder iterable byte queueing strategy trxcllnt Jun 6, 2019ba755ad use 53 bit hash fn to further avoid collisions trxcllnt Jun 6, 20199c3a865 ensure byteLength is calculated for offsets buffer trxcllnt Jun 6, 2019c0f7f7b fix a few minor formatting issues in arrow2csv trxcllnt Jun 7, 201905acad8 fix minor row serialization issues to be compatible with console.table() trxcllnt Jun 7, 20194d2a4f0 regenerate flatbuffer source from current format schemas trxcllnt Jun 7, 20198b0752f fix possible AsyncRandomAccessFile race condition retrieving filehand… trxcllnt Jun 7, 201907fa443 remove default dictionary hash function trxcllnt Jun 19, 20197998d2a add createIsValidFunction example docstring trxcllnt Jun 19, 2019 File filter Filter by extension Filter by extension .js (5) .json (4) .ts (117) All 3 file types selected Only manifest files Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace js/.vscode/launch.json launch.json js/bin/print-buffer-alignment.js print-buffer-alignment.js js/gulp/closure-task.js closure-task.js js/gulp/test-task.js test-task.js js/gulp/util.js util.js js/package-lock.json package-lock.json js/package.json package.json js/src/Arrow.dom.ts Arrow.dom.ts js/src/Arrow.node.ts Arrow.node.ts js/src/Arrow.ts Arrow.ts js/src/bin/arrow2csv.ts arrow2csv.ts js/src/builder.ts builder.ts js/src/builder/binary.ts binary.ts js/src/builder/bool.ts bool.ts js/src/builder/buffer.ts buffer.ts js/src/builder/date.ts date.ts js/src/builder/decimal.ts decimal.ts js/src/builder/dictionary.ts dictionary.ts js/src/builder/fixedsizebinary.ts fixedsizebinary.ts js/src/builder/fixedsizelist.ts fixedsizelist.ts js/src/builder/float.ts float.ts js/src/builder/index.ts index.ts js/src/builder/int.ts int.ts js/src/builder/interval.ts interval.ts js/src/builder/list.ts list.ts js/src/builder/map.ts map.ts js/src/builder/null.ts null.ts js/src/builder/run.ts run.ts js/src/builder/struct.ts struct.ts js/src/builder/time.ts time.ts js/src/builder/timestamp.ts timestamp.ts js/src/builder/union.ts union.ts js/src/builder/utf8.ts utf8.ts js/src/builder/valid.ts valid.ts js/src/column.ts column.ts js/src/compute/dataframe.ts dataframe.ts js/src/data.ts data.ts js/src/enum.ts enum.ts js/src/fb/File.ts File.ts js/src/fb/Message.ts Message.ts js/src/fb/Schema.ts Schema.ts js/src/interfaces.ts interfaces.ts js/src/io/adapters.ts adapters.ts js/src/io/file.ts file.ts js/src/io/node/builder.ts builder.ts js/src/io/node/iterable.ts iterable.ts js/src/io/node/reader.ts reader.ts js/src/io/node/writer.ts writer.ts js/src/io/stream.ts stream.ts js/src/io/whatwg/builder.ts builder.ts js/src/io/whatwg/iterable.ts iterable.ts js/src/io/whatwg/reader.ts reader.ts js/src/io/whatwg/writer.ts writer.ts js/src/ipc/metadata/json.ts json.ts js/src/ipc/metadata/message.ts message.ts js/src/ipc/writer.ts writer.ts js/src/recordbatch.ts recordbatch.ts js/src/schema.ts schema.ts js/src/table.ts table.ts js/src/type.ts type.ts js/src/util/args.ts args.ts js/src/util/bn.ts bn.ts js/src/util/buffer.ts buffer.ts js/src/util/compat.ts compat.ts js/src/util/fn.ts fn.ts js/src/util/int.ts int.ts js/src/util/utf8.ts utf8.ts js/src/util/vector.ts vector.ts js/src/vector.ts vector.ts js/src/vector/base.ts base.ts js/src/vector/binary.ts binary.ts js/src/vector/bool.ts bool.ts js/src/vector/chunked.ts chunked.ts js/src/vector/date.ts date.ts js/src/vector/decimal.ts decimal.ts js/src/vector/dictionary.ts dictionary.ts js/src/vector/fixedsizebinary.ts fixedsizebinary.ts js/src/vector/fixedsizelist.ts fixedsizelist.ts js/src/vector/float.ts float.ts js/src/vector/index.ts index.ts js/src/vector/int.ts int.ts js/src/vector/interval.ts interval.ts js/src/vector/list.ts list.ts js/src/vector/map.ts map.ts js/src/vector/null.ts null.ts js/src/vector/row.ts row.ts js/src/vector/struct.ts struct.ts js/src/vector/time.ts time.ts js/src/vector/timestamp.ts timestamp.ts js/src/vector/union.ts union.ts js/src/vector/utf8.ts utf8.ts js/src/visitor/builderctor.ts builderctor.ts js/src/visitor/bytewidth.ts bytewidth.ts js/src/visitor/get.ts get.ts js/src/visitor/indexof.ts indexof.ts js/src/visitor/iterator.ts iterator.ts js/src/visitor/jsontypeassembler.ts jsontypeassembler.ts js/src/visitor/jsonvectorassembler.ts jsonvectorassembler.ts js/src/visitor/set.ts set.ts js/src/visitor/toarray.ts toarray.ts js/src/visitor/typeassembler.ts typeassembler.ts js/src/visitor/typecomparator.ts typecomparator.ts js/src/visitor/typector.ts typector.ts js/src/visitor/vectorassembler.ts vectorassembler.ts js/src/visitor/vectorctor.ts vectorctor.ts js/src/visitor/vectorloader.ts vectorloader.ts js/test/Arrow.ts Arrow.ts js/test/generate-test-data.ts generate-test-data.ts js/test/inference/column.ts column.ts js/test/inference/nested.ts nested.ts js/test/jest-extensions.ts jest-extensions.ts js/test/unit/builders/builder-tests.ts builder-tests.ts js/test/unit/builders/date-tests.ts date-tests.ts js/test/unit/builders/dictionary-tests.ts dictionary-tests.ts js/test/unit/builders/int64-tests.ts int64-tests.ts js/test/unit/builders/primitive-tests.ts primitive-tests.ts js/test/unit/builders/uint64-tests.ts uint64-tests.ts js/test/unit/builders/utf8-tests.ts utf8-tests.ts js/test/unit/builders/utils.ts utils.ts js/test/unit/generated-data-tests.ts generated-data-tests.ts js/test/unit/ipc/writer/streams-dom-tests.ts streams-dom-tests.ts js/test/unit/ipc/writer/streams-node-tests.ts streams-node-tests.ts js/test/unit/vector/bool-vector-tests.ts bool-vector-tests.ts js/test/unit/vector/numeric-vector-tests.ts numeric-vector-tests.ts js/tsconfig/tsconfig.base.json tsconfig.base.json js/typedoc.js typedoc.js There are no files selected for viewing 11 changes: 10 additions & 1 deletion 11 js/.vscode/launch.json Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -37,8 +37,17 @@ // "TEST_MODULE": "umd" }, "args": [ // "-i", "-i", "test/unit/", // "test/unit/builders/", // "test/unit/builders/builder-tests.ts", // "test/unit/builders/int64-tests.ts", // "test/unit/builders/uint64-tests.ts", // "test/unit/builders/date-tests.ts", // "test/unit/builders/primitive-tests.ts", // "test/unit/builders/dictionary-tests.ts", // "test/unit/builders/utf8-tests.ts", // Uncomment any of these to run individual test suites // "test/unit/int-tests.ts", Expand Down 36 changes: 30 additions & 6 deletions 36 js/bin/print-buffer-alignment.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,29 +22,45 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; const { AsyncMessageReader } = require(`../index${extension}`); const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); let recordBatchIndex = 0, dictionaryBatchIndex = 0; let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; for await (let message of reader) { let bufferRegions = []; if (message.isSchema()) { schema = message.header(); continue; } else if (message.isRecordBatch()) { bufferRegions = message.header().buffers; const header = message.header(); bufferRegions = header.buffers; const body = await reader.readMessageBody(message.bodyLength); console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadRecordBatch(schema, header, body); console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, })}`); } else if (message.isDictionaryBatch()) { bufferRegions = message.header().data.buffers; const header = message.header(); bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); const body = await reader.readMessageBody(message.bodyLength); console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadDictionaryBatch(header.data, body, type); console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, dictionaryId: header.id, })}`); } bufferRegions.forEach(({ offset, length }, i) => { Expand All @@ -55,3 +71,11 @@ const { AsyncMessageReader } = require(`../index${extension}`); await reader.return(); })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields)); } function loadDictionaryBatch(header, body, dictionaryType) { return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType])); } 1 change: 0 additions & 1 deletion 1 js/gulp/closure-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ // formatting: `PRETTY_PRINT`, // debug: true, compilation_level: `ADVANCED`, allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/test-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.verbose && jestArgv.push(`--verbose`); argv.coverage ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`) ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`) : jestArgv.push(`-c`, `jest.config.js`, `-i`) const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/util.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) { env: { ...process.env, NODE_NO_WARNINGS: `1` } }; return asyncDone(() => child_process.spawn(`node`, args, opts)) .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; }); .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); } const logAndDie = (e) => { if (e) { process.exit(1); } }; Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /arrowPublic Notifications You must be signed in to change notification settings Fork 4.3k Star 17.1k Code Issues2.2k Pull requests401 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Closed trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders Conversation 23 Commits 56 Checks 0 Files changed Uh oh! There was an error while loading. Please reload this page. Closed ARROW-5115: [JS] Add Vector Builders and high-level stream primitives #4476 Show file tree Hide file tree Changes from all commits Commits Show all changes 56 commits Select commit Hold shift + click to select a range 738fabb WIP streaming data builders trxcllnt Feb 1, 2019ff32cbb fix date builder tests trxcllnt Feb 3, 20199e5e3ea fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196ebaa36 WIP streaming data builders trxcllnt Feb 1, 20192f423ec fix date builder tests trxcllnt Feb 3, 2019c609266 fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196b7a20e ensure 64-bit int builders use BN to check for nulls trxcllnt Feb 6, 20198a5c713 show a better error message if gulp fails trxcllnt Mar 20, 2019cf5f778 return signed or unsigned 53bit integers trxcllnt Mar 20, 20196e64c40 ensure builder allocates aligned buffers, null bitmaps initialized to… trxcllnt Mar 20, 20192b7b992 ensure union typeids buffer is an Int8Array trxcllnt Mar 20, 2019f7fe8c1 update builder buffer padding trxcllnt Apr 7, 20191398ebd don't clone Dictionary DataType instances in Schema assign to preserv… trxcllnt Apr 8, 2019fb2c9a2 cleanup trxcllnt Apr 9, 20192f36741 move stream methods to io folder trxcllnt Apr 15, 2019385fee3 fix typo trxcllnt Apr 15, 2019d193141 add helper method to return the stride for a datatype trxcllnt Apr 15, 201970377a1 add vectorname getter to chunked for completion trxcllnt Apr 15, 2019f8760cb enumerate each type key individually now that they're a real thing trxcllnt Apr 15, 20199ba9965 update row type inference sanity check trxcllnt Apr 15, 20194c2ef42 add the rest of the builder types trxcllnt Apr 15, 2019a307273 fix readable-stream detection in firefox trxcllnt Apr 16, 2019f651657 update typescript, ts-jest, jest trxcllnt Apr 16, 2019f708190 use safe BigInt64Array constructor trxcllnt Apr 17, 201915ac17c use Object.setPrototypeOf to improve bn performance trxcllnt May 11, 20190990344 add Builder throughDOM and throughNode transform streams trxcllnt May 17, 20192b7a63c update test types trxcllnt May 22, 201979bfd46 fix node and dom builder streams trxcllnt May 22, 201926387c7 update typescript, finish streaming builders, add comprehensive build… trxcllnt May 27, 2019fd46163 fix nan checks trxcllnt May 27, 2019d032e2d update dependencies trxcllnt May 27, 201986a6cd0 update closure compiler dependency trxcllnt May 27, 2019c587c33 add BufferBuilders, clean up public Builder API trxcllnt May 29, 20190209e9d update to typescript 3.5 trxcllnt May 30, 201984beb68 remove some property getters, clean up trxcllnt May 30, 2019dddba1a ensure variablewidthbuilder supports random insertion trxcllnt May 31, 2019893c74f ensure ListBuilder supports random insertion trxcllnt Jun 1, 2019c2673ac add initial builder jsdoc trxcllnt Jun 4, 20195b990c3 Clean up typedoc output, update typedoc to master to use typescript@3… trxcllnt Jun 5, 2019b2556b7 add docstring for _InternalEmptyPlaceholderRecordBatch trxcllnt Jun 5, 20194b5375e remove unnecessary jsdoc trxcllnt Jun 5, 20195d4b0be ensure bitmapbufferbuilder increments and decrements _popCount approp… trxcllnt Jun 5, 2019e974444 fix lint trxcllnt Jun 5, 20195aea4f9 Add more specific Int64 and Uint64 Builder tests trxcllnt Jun 5, 20193adf555 adds or updates most of the high-level Vector.from() methods to use t… trxcllnt Jun 6, 2019f250845 use a better default dictionary builder hash function trxcllnt Jun 6, 201913b51db print more details about each message trxcllnt Jun 6, 20198659323 add test for builder iterable byte queueing strategy trxcllnt Jun 6, 2019ba755ad use 53 bit hash fn to further avoid collisions trxcllnt Jun 6, 20199c3a865 ensure byteLength is calculated for offsets buffer trxcllnt Jun 6, 2019c0f7f7b fix a few minor formatting issues in arrow2csv trxcllnt Jun 7, 201905acad8 fix minor row serialization issues to be compatible with console.table() trxcllnt Jun 7, 20194d2a4f0 regenerate flatbuffer source from current format schemas trxcllnt Jun 7, 20198b0752f fix possible AsyncRandomAccessFile race condition retrieving filehand… trxcllnt Jun 7, 201907fa443 remove default dictionary hash function trxcllnt Jun 19, 20197998d2a add createIsValidFunction example docstring trxcllnt Jun 19, 2019 File filter Filter by extension Filter by extension .js (5) .json (4) .ts (117) All 3 file types selected Only manifest files Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace js/.vscode/launch.json launch.json js/bin/print-buffer-alignment.js print-buffer-alignment.js js/gulp/closure-task.js closure-task.js js/gulp/test-task.js test-task.js js/gulp/util.js util.js js/package-lock.json package-lock.json js/package.json package.json js/src/Arrow.dom.ts Arrow.dom.ts js/src/Arrow.node.ts Arrow.node.ts js/src/Arrow.ts Arrow.ts js/src/bin/arrow2csv.ts arrow2csv.ts js/src/builder.ts builder.ts js/src/builder/binary.ts binary.ts js/src/builder/bool.ts bool.ts js/src/builder/buffer.ts buffer.ts js/src/builder/date.ts date.ts js/src/builder/decimal.ts decimal.ts js/src/builder/dictionary.ts dictionary.ts js/src/builder/fixedsizebinary.ts fixedsizebinary.ts js/src/builder/fixedsizelist.ts fixedsizelist.ts js/src/builder/float.ts float.ts js/src/builder/index.ts index.ts js/src/builder/int.ts int.ts js/src/builder/interval.ts interval.ts js/src/builder/list.ts list.ts js/src/builder/map.ts map.ts js/src/builder/null.ts null.ts js/src/builder/run.ts run.ts js/src/builder/struct.ts struct.ts js/src/builder/time.ts time.ts js/src/builder/timestamp.ts timestamp.ts js/src/builder/union.ts union.ts js/src/builder/utf8.ts utf8.ts js/src/builder/valid.ts valid.ts js/src/column.ts column.ts js/src/compute/dataframe.ts dataframe.ts js/src/data.ts data.ts js/src/enum.ts enum.ts js/src/fb/File.ts File.ts js/src/fb/Message.ts Message.ts js/src/fb/Schema.ts Schema.ts js/src/interfaces.ts interfaces.ts js/src/io/adapters.ts adapters.ts js/src/io/file.ts file.ts js/src/io/node/builder.ts builder.ts js/src/io/node/iterable.ts iterable.ts js/src/io/node/reader.ts reader.ts js/src/io/node/writer.ts writer.ts js/src/io/stream.ts stream.ts js/src/io/whatwg/builder.ts builder.ts js/src/io/whatwg/iterable.ts iterable.ts js/src/io/whatwg/reader.ts reader.ts js/src/io/whatwg/writer.ts writer.ts js/src/ipc/metadata/json.ts json.ts js/src/ipc/metadata/message.ts message.ts js/src/ipc/writer.ts writer.ts js/src/recordbatch.ts recordbatch.ts js/src/schema.ts schema.ts js/src/table.ts table.ts js/src/type.ts type.ts js/src/util/args.ts args.ts js/src/util/bn.ts bn.ts js/src/util/buffer.ts buffer.ts js/src/util/compat.ts compat.ts js/src/util/fn.ts fn.ts js/src/util/int.ts int.ts js/src/util/utf8.ts utf8.ts js/src/util/vector.ts vector.ts js/src/vector.ts vector.ts js/src/vector/base.ts base.ts js/src/vector/binary.ts binary.ts js/src/vector/bool.ts bool.ts js/src/vector/chunked.ts chunked.ts js/src/vector/date.ts date.ts js/src/vector/decimal.ts decimal.ts js/src/vector/dictionary.ts dictionary.ts js/src/vector/fixedsizebinary.ts fixedsizebinary.ts js/src/vector/fixedsizelist.ts fixedsizelist.ts js/src/vector/float.ts float.ts js/src/vector/index.ts index.ts js/src/vector/int.ts int.ts js/src/vector/interval.ts interval.ts js/src/vector/list.ts list.ts js/src/vector/map.ts map.ts js/src/vector/null.ts null.ts js/src/vector/row.ts row.ts js/src/vector/struct.ts struct.ts js/src/vector/time.ts time.ts js/src/vector/timestamp.ts timestamp.ts js/src/vector/union.ts union.ts js/src/vector/utf8.ts utf8.ts js/src/visitor/builderctor.ts builderctor.ts js/src/visitor/bytewidth.ts bytewidth.ts js/src/visitor/get.ts get.ts js/src/visitor/indexof.ts indexof.ts js/src/visitor/iterator.ts iterator.ts js/src/visitor/jsontypeassembler.ts jsontypeassembler.ts js/src/visitor/jsonvectorassembler.ts jsonvectorassembler.ts js/src/visitor/set.ts set.ts js/src/visitor/toarray.ts toarray.ts js/src/visitor/typeassembler.ts typeassembler.ts js/src/visitor/typecomparator.ts typecomparator.ts js/src/visitor/typector.ts typector.ts js/src/visitor/vectorassembler.ts vectorassembler.ts js/src/visitor/vectorctor.ts vectorctor.ts js/src/visitor/vectorloader.ts vectorloader.ts js/test/Arrow.ts Arrow.ts js/test/generate-test-data.ts generate-test-data.ts js/test/inference/column.ts column.ts js/test/inference/nested.ts nested.ts js/test/jest-extensions.ts jest-extensions.ts js/test/unit/builders/builder-tests.ts builder-tests.ts js/test/unit/builders/date-tests.ts date-tests.ts js/test/unit/builders/dictionary-tests.ts dictionary-tests.ts js/test/unit/builders/int64-tests.ts int64-tests.ts js/test/unit/builders/primitive-tests.ts primitive-tests.ts js/test/unit/builders/uint64-tests.ts uint64-tests.ts js/test/unit/builders/utf8-tests.ts utf8-tests.ts js/test/unit/builders/utils.ts utils.ts js/test/unit/generated-data-tests.ts generated-data-tests.ts js/test/unit/ipc/writer/streams-dom-tests.ts streams-dom-tests.ts js/test/unit/ipc/writer/streams-node-tests.ts streams-node-tests.ts js/test/unit/vector/bool-vector-tests.ts bool-vector-tests.ts js/test/unit/vector/numeric-vector-tests.ts numeric-vector-tests.ts js/tsconfig/tsconfig.base.json tsconfig.base.json js/typedoc.js typedoc.js There are no files selected for viewing 11 changes: 10 additions & 1 deletion 11 js/.vscode/launch.json Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -37,8 +37,17 @@ // "TEST_MODULE": "umd" }, "args": [ // "-i", "-i", "test/unit/", // "test/unit/builders/", // "test/unit/builders/builder-tests.ts", // "test/unit/builders/int64-tests.ts", // "test/unit/builders/uint64-tests.ts", // "test/unit/builders/date-tests.ts", // "test/unit/builders/primitive-tests.ts", // "test/unit/builders/dictionary-tests.ts", // "test/unit/builders/utf8-tests.ts", // Uncomment any of these to run individual test suites // "test/unit/int-tests.ts", Expand Down 36 changes: 30 additions & 6 deletions 36 js/bin/print-buffer-alignment.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,29 +22,45 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; const { AsyncMessageReader } = require(`../index${extension}`); const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); let recordBatchIndex = 0, dictionaryBatchIndex = 0; let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; for await (let message of reader) { let bufferRegions = []; if (message.isSchema()) { schema = message.header(); continue; } else if (message.isRecordBatch()) { bufferRegions = message.header().buffers; const header = message.header(); bufferRegions = header.buffers; const body = await reader.readMessageBody(message.bodyLength); console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadRecordBatch(schema, header, body); console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, })}`); } else if (message.isDictionaryBatch()) { bufferRegions = message.header().data.buffers; const header = message.header(); bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); const body = await reader.readMessageBody(message.bodyLength); console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadDictionaryBatch(header.data, body, type); console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, dictionaryId: header.id, })}`); } bufferRegions.forEach(({ offset, length }, i) => { Expand All @@ -55,3 +71,11 @@ const { AsyncMessageReader } = require(`../index${extension}`); await reader.return(); })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields)); } function loadDictionaryBatch(header, body, dictionaryType) { return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType])); } 1 change: 0 additions & 1 deletion 1 js/gulp/closure-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ // formatting: `PRETTY_PRINT`, // debug: true, compilation_level: `ADVANCED`, allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/test-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.verbose && jestArgv.push(`--verbose`); argv.coverage ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`) ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`) : jestArgv.push(`-c`, `jest.config.js`, `-i`) const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/util.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) { env: { ...process.env, NODE_NO_WARNINGS: `1` } }; return asyncDone(() => child_process.spawn(`node`, args, opts)) .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; }); .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); } const logAndDie = (e) => { if (e) { process.exit(1); } }; Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /arrowPublic Notifications You must be signed in to change notification settings Fork 4.3k Star 17.1k Code Issues2.2k Pull requests401 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Closed trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders Conversation 23 Commits 56 Checks 0 Files changed Uh oh! There was an error while loading. Please reload this page. Closed ARROW-5115: [JS] Add Vector Builders and high-level stream primitives #4476 Show file tree Hide file tree Changes from all commits Commits Show all changes 56 commits Select commit Hold shift + click to select a range 738fabb WIP streaming data builders trxcllnt Feb 1, 2019ff32cbb fix date builder tests trxcllnt Feb 3, 20199e5e3ea fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196ebaa36 WIP streaming data builders trxcllnt Feb 1, 20192f423ec fix date builder tests trxcllnt Feb 3, 2019c609266 fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196b7a20e ensure 64-bit int builders use BN to check for nulls trxcllnt Feb 6, 20198a5c713 show a better error message if gulp fails trxcllnt Mar 20, 2019cf5f778 return signed or unsigned 53bit integers trxcllnt Mar 20, 20196e64c40 ensure builder allocates aligned buffers, null bitmaps initialized to… trxcllnt Mar 20, 20192b7b992 ensure union typeids buffer is an Int8Array trxcllnt Mar 20, 2019f7fe8c1 update builder buffer padding trxcllnt Apr 7, 20191398ebd don't clone Dictionary DataType instances in Schema assign to preserv… trxcllnt Apr 8, 2019fb2c9a2 cleanup trxcllnt Apr 9, 20192f36741 move stream methods to io folder trxcllnt Apr 15, 2019385fee3 fix typo trxcllnt Apr 15, 2019d193141 add helper method to return the stride for a datatype trxcllnt Apr 15, 201970377a1 add vectorname getter to chunked for completion trxcllnt Apr 15, 2019f8760cb enumerate each type key individually now that they're a real thing trxcllnt Apr 15, 20199ba9965 update row type inference sanity check trxcllnt Apr 15, 20194c2ef42 add the rest of the builder types trxcllnt Apr 15, 2019a307273 fix readable-stream detection in firefox trxcllnt Apr 16, 2019f651657 update typescript, ts-jest, jest trxcllnt Apr 16, 2019f708190 use safe BigInt64Array constructor trxcllnt Apr 17, 201915ac17c use Object.setPrototypeOf to improve bn performance trxcllnt May 11, 20190990344 add Builder throughDOM and throughNode transform streams trxcllnt May 17, 20192b7a63c update test types trxcllnt May 22, 201979bfd46 fix node and dom builder streams trxcllnt May 22, 201926387c7 update typescript, finish streaming builders, add comprehensive build… trxcllnt May 27, 2019fd46163 fix nan checks trxcllnt May 27, 2019d032e2d update dependencies trxcllnt May 27, 201986a6cd0 update closure compiler dependency trxcllnt May 27, 2019c587c33 add BufferBuilders, clean up public Builder API trxcllnt May 29, 20190209e9d update to typescript 3.5 trxcllnt May 30, 201984beb68 remove some property getters, clean up trxcllnt May 30, 2019dddba1a ensure variablewidthbuilder supports random insertion trxcllnt May 31, 2019893c74f ensure ListBuilder supports random insertion trxcllnt Jun 1, 2019c2673ac add initial builder jsdoc trxcllnt Jun 4, 20195b990c3 Clean up typedoc output, update typedoc to master to use typescript@3… trxcllnt Jun 5, 2019b2556b7 add docstring for _InternalEmptyPlaceholderRecordBatch trxcllnt Jun 5, 20194b5375e remove unnecessary jsdoc trxcllnt Jun 5, 20195d4b0be ensure bitmapbufferbuilder increments and decrements _popCount approp… trxcllnt Jun 5, 2019e974444 fix lint trxcllnt Jun 5, 20195aea4f9 Add more specific Int64 and Uint64 Builder tests trxcllnt Jun 5, 20193adf555 adds or updates most of the high-level Vector.from() methods to use t… trxcllnt Jun 6, 2019f250845 use a better default dictionary builder hash function trxcllnt Jun 6, 201913b51db print more details about each message trxcllnt Jun 6, 20198659323 add test for builder iterable byte queueing strategy trxcllnt Jun 6, 2019ba755ad use 53 bit hash fn to further avoid collisions trxcllnt Jun 6, 20199c3a865 ensure byteLength is calculated for offsets buffer trxcllnt Jun 6, 2019c0f7f7b fix a few minor formatting issues in arrow2csv trxcllnt Jun 7, 201905acad8 fix minor row serialization issues to be compatible with console.table() trxcllnt Jun 7, 20194d2a4f0 regenerate flatbuffer source from current format schemas trxcllnt Jun 7, 20198b0752f fix possible AsyncRandomAccessFile race condition retrieving filehand… trxcllnt Jun 7, 201907fa443 remove default dictionary hash function trxcllnt Jun 19, 20197998d2a add createIsValidFunction example docstring trxcllnt Jun 19, 2019 File filter Filter by extension Filter by extension .js (5) .json (4) .ts (117) All 3 file types selected Only manifest files Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace js/.vscode/launch.json launch.json js/bin/print-buffer-alignment.js print-buffer-alignment.js js/gulp/closure-task.js closure-task.js js/gulp/test-task.js test-task.js js/gulp/util.js util.js js/package-lock.json package-lock.json js/package.json package.json js/src/Arrow.dom.ts Arrow.dom.ts js/src/Arrow.node.ts Arrow.node.ts js/src/Arrow.ts Arrow.ts js/src/bin/arrow2csv.ts arrow2csv.ts js/src/builder.ts builder.ts js/src/builder/binary.ts binary.ts js/src/builder/bool.ts bool.ts js/src/builder/buffer.ts buffer.ts js/src/builder/date.ts date.ts js/src/builder/decimal.ts decimal.ts js/src/builder/dictionary.ts dictionary.ts js/src/builder/fixedsizebinary.ts fixedsizebinary.ts js/src/builder/fixedsizelist.ts fixedsizelist.ts js/src/builder/float.ts float.ts js/src/builder/index.ts index.ts js/src/builder/int.ts int.ts js/src/builder/interval.ts interval.ts js/src/builder/list.ts list.ts js/src/builder/map.ts map.ts js/src/builder/null.ts null.ts js/src/builder/run.ts run.ts js/src/builder/struct.ts struct.ts js/src/builder/time.ts time.ts js/src/builder/timestamp.ts timestamp.ts js/src/builder/union.ts union.ts js/src/builder/utf8.ts utf8.ts js/src/builder/valid.ts valid.ts js/src/column.ts column.ts js/src/compute/dataframe.ts dataframe.ts js/src/data.ts data.ts js/src/enum.ts enum.ts js/src/fb/File.ts File.ts js/src/fb/Message.ts Message.ts js/src/fb/Schema.ts Schema.ts js/src/interfaces.ts interfaces.ts js/src/io/adapters.ts adapters.ts js/src/io/file.ts file.ts js/src/io/node/builder.ts builder.ts js/src/io/node/iterable.ts iterable.ts js/src/io/node/reader.ts reader.ts js/src/io/node/writer.ts writer.ts js/src/io/stream.ts stream.ts js/src/io/whatwg/builder.ts builder.ts js/src/io/whatwg/iterable.ts iterable.ts js/src/io/whatwg/reader.ts reader.ts js/src/io/whatwg/writer.ts writer.ts js/src/ipc/metadata/json.ts json.ts js/src/ipc/metadata/message.ts message.ts js/src/ipc/writer.ts writer.ts js/src/recordbatch.ts recordbatch.ts js/src/schema.ts schema.ts js/src/table.ts table.ts js/src/type.ts type.ts js/src/util/args.ts args.ts js/src/util/bn.ts bn.ts js/src/util/buffer.ts buffer.ts js/src/util/compat.ts compat.ts js/src/util/fn.ts fn.ts js/src/util/int.ts int.ts js/src/util/utf8.ts utf8.ts js/src/util/vector.ts vector.ts js/src/vector.ts vector.ts js/src/vector/base.ts base.ts js/src/vector/binary.ts binary.ts js/src/vector/bool.ts bool.ts js/src/vector/chunked.ts chunked.ts js/src/vector/date.ts date.ts js/src/vector/decimal.ts decimal.ts js/src/vector/dictionary.ts dictionary.ts js/src/vector/fixedsizebinary.ts fixedsizebinary.ts js/src/vector/fixedsizelist.ts fixedsizelist.ts js/src/vector/float.ts float.ts js/src/vector/index.ts index.ts js/src/vector/int.ts int.ts js/src/vector/interval.ts interval.ts js/src/vector/list.ts list.ts js/src/vector/map.ts map.ts js/src/vector/null.ts null.ts js/src/vector/row.ts row.ts js/src/vector/struct.ts struct.ts js/src/vector/time.ts time.ts js/src/vector/timestamp.ts timestamp.ts js/src/vector/union.ts union.ts js/src/vector/utf8.ts utf8.ts js/src/visitor/builderctor.ts builderctor.ts js/src/visitor/bytewidth.ts bytewidth.ts js/src/visitor/get.ts get.ts js/src/visitor/indexof.ts indexof.ts js/src/visitor/iterator.ts iterator.ts js/src/visitor/jsontypeassembler.ts jsontypeassembler.ts js/src/visitor/jsonvectorassembler.ts jsonvectorassembler.ts js/src/visitor/set.ts set.ts js/src/visitor/toarray.ts toarray.ts js/src/visitor/typeassembler.ts typeassembler.ts js/src/visitor/typecomparator.ts typecomparator.ts js/src/visitor/typector.ts typector.ts js/src/visitor/vectorassembler.ts vectorassembler.ts js/src/visitor/vectorctor.ts vectorctor.ts js/src/visitor/vectorloader.ts vectorloader.ts js/test/Arrow.ts Arrow.ts js/test/generate-test-data.ts generate-test-data.ts js/test/inference/column.ts column.ts js/test/inference/nested.ts nested.ts js/test/jest-extensions.ts jest-extensions.ts js/test/unit/builders/builder-tests.ts builder-tests.ts js/test/unit/builders/date-tests.ts date-tests.ts js/test/unit/builders/dictionary-tests.ts dictionary-tests.ts js/test/unit/builders/int64-tests.ts int64-tests.ts js/test/unit/builders/primitive-tests.ts primitive-tests.ts js/test/unit/builders/uint64-tests.ts uint64-tests.ts js/test/unit/builders/utf8-tests.ts utf8-tests.ts js/test/unit/builders/utils.ts utils.ts js/test/unit/generated-data-tests.ts generated-data-tests.ts js/test/unit/ipc/writer/streams-dom-tests.ts streams-dom-tests.ts js/test/unit/ipc/writer/streams-node-tests.ts streams-node-tests.ts js/test/unit/vector/bool-vector-tests.ts bool-vector-tests.ts js/test/unit/vector/numeric-vector-tests.ts numeric-vector-tests.ts js/tsconfig/tsconfig.base.json tsconfig.base.json js/typedoc.js typedoc.js There are no files selected for viewing 11 changes: 10 additions & 1 deletion 11 js/.vscode/launch.json Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -37,8 +37,17 @@ // "TEST_MODULE": "umd" }, "args": [ // "-i", "-i", "test/unit/", // "test/unit/builders/", // "test/unit/builders/builder-tests.ts", // "test/unit/builders/int64-tests.ts", // "test/unit/builders/uint64-tests.ts", // "test/unit/builders/date-tests.ts", // "test/unit/builders/primitive-tests.ts", // "test/unit/builders/dictionary-tests.ts", // "test/unit/builders/utf8-tests.ts", // Uncomment any of these to run individual test suites // "test/unit/int-tests.ts", Expand Down 36 changes: 30 additions & 6 deletions 36 js/bin/print-buffer-alignment.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,29 +22,45 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; const { AsyncMessageReader } = require(`../index${extension}`); const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); let recordBatchIndex = 0, dictionaryBatchIndex = 0; let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; for await (let message of reader) { let bufferRegions = []; if (message.isSchema()) { schema = message.header(); continue; } else if (message.isRecordBatch()) { bufferRegions = message.header().buffers; const header = message.header(); bufferRegions = header.buffers; const body = await reader.readMessageBody(message.bodyLength); console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadRecordBatch(schema, header, body); console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, })}`); } else if (message.isDictionaryBatch()) { bufferRegions = message.header().data.buffers; const header = message.header(); bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); const body = await reader.readMessageBody(message.bodyLength); console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadDictionaryBatch(header.data, body, type); console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, dictionaryId: header.id, })}`); } bufferRegions.forEach(({ offset, length }, i) => { Expand All @@ -55,3 +71,11 @@ const { AsyncMessageReader } = require(`../index${extension}`); await reader.return(); })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields)); } function loadDictionaryBatch(header, body, dictionaryType) { return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType])); } 1 change: 0 additions & 1 deletion 1 js/gulp/closure-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ // formatting: `PRETTY_PRINT`, // debug: true, compilation_level: `ADVANCED`, allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/test-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.verbose && jestArgv.push(`--verbose`); argv.coverage ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`) ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`) : jestArgv.push(`-c`, `jest.config.js`, `-i`) const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/util.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) { env: { ...process.env, NODE_NO_WARNINGS: `1` } }; return asyncDone(() => child_process.spawn(`node`, args, opts)) .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; }); .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); } const logAndDie = (e) => { if (e) { process.exit(1); } }; Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /arrowPublic Notifications You must be signed in to change notification settings Fork 4.3k Star 17.1k Code Issues2.2k Pull requests401 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Closed trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders Conversation 23 Commits 56 Checks 0 Files changed Uh oh! There was an error while loading. Please reload this page. Closed ARROW-5115: [JS] Add Vector Builders and high-level stream primitives #4476 Show file tree Hide file tree Changes from all commits Commits Show all changes 56 commits Select commit Hold shift + click to select a range 738fabb WIP streaming data builders trxcllnt Feb 1, 2019ff32cbb fix date builder tests trxcllnt Feb 3, 20199e5e3ea fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196ebaa36 WIP streaming data builders trxcllnt Feb 1, 20192f423ec fix date builder tests trxcllnt Feb 3, 2019c609266 fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196b7a20e ensure 64-bit int builders use BN to check for nulls trxcllnt Feb 6, 20198a5c713 show a better error message if gulp fails trxcllnt Mar 20, 2019cf5f778 return signed or unsigned 53bit integers trxcllnt Mar 20, 20196e64c40 ensure builder allocates aligned buffers, null bitmaps initialized to… trxcllnt Mar 20, 20192b7b992 ensure union typeids buffer is an Int8Array trxcllnt Mar 20, 2019f7fe8c1 update builder buffer padding trxcllnt Apr 7, 20191398ebd don't clone Dictionary DataType instances in Schema assign to preserv… trxcllnt Apr 8, 2019fb2c9a2 cleanup trxcllnt Apr 9, 20192f36741 move stream methods to io folder trxcllnt Apr 15, 2019385fee3 fix typo trxcllnt Apr 15, 2019d193141 add helper method to return the stride for a datatype trxcllnt Apr 15, 201970377a1 add vectorname getter to chunked for completion trxcllnt Apr 15, 2019f8760cb enumerate each type key individually now that they're a real thing trxcllnt Apr 15, 20199ba9965 update row type inference sanity check trxcllnt Apr 15, 20194c2ef42 add the rest of the builder types trxcllnt Apr 15, 2019a307273 fix readable-stream detection in firefox trxcllnt Apr 16, 2019f651657 update typescript, ts-jest, jest trxcllnt Apr 16, 2019f708190 use safe BigInt64Array constructor trxcllnt Apr 17, 201915ac17c use Object.setPrototypeOf to improve bn performance trxcllnt May 11, 20190990344 add Builder throughDOM and throughNode transform streams trxcllnt May 17, 20192b7a63c update test types trxcllnt May 22, 201979bfd46 fix node and dom builder streams trxcllnt May 22, 201926387c7 update typescript, finish streaming builders, add comprehensive build… trxcllnt May 27, 2019fd46163 fix nan checks trxcllnt May 27, 2019d032e2d update dependencies trxcllnt May 27, 201986a6cd0 update closure compiler dependency trxcllnt May 27, 2019c587c33 add BufferBuilders, clean up public Builder API trxcllnt May 29, 20190209e9d update to typescript 3.5 trxcllnt May 30, 201984beb68 remove some property getters, clean up trxcllnt May 30, 2019dddba1a ensure variablewidthbuilder supports random insertion trxcllnt May 31, 2019893c74f ensure ListBuilder supports random insertion trxcllnt Jun 1, 2019c2673ac add initial builder jsdoc trxcllnt Jun 4, 20195b990c3 Clean up typedoc output, update typedoc to master to use typescript@3… trxcllnt Jun 5, 2019b2556b7 add docstring for _InternalEmptyPlaceholderRecordBatch trxcllnt Jun 5, 20194b5375e remove unnecessary jsdoc trxcllnt Jun 5, 20195d4b0be ensure bitmapbufferbuilder increments and decrements _popCount approp… trxcllnt Jun 5, 2019e974444 fix lint trxcllnt Jun 5, 20195aea4f9 Add more specific Int64 and Uint64 Builder tests trxcllnt Jun 5, 20193adf555 adds or updates most of the high-level Vector.from() methods to use t… trxcllnt Jun 6, 2019f250845 use a better default dictionary builder hash function trxcllnt Jun 6, 201913b51db print more details about each message trxcllnt Jun 6, 20198659323 add test for builder iterable byte queueing strategy trxcllnt Jun 6, 2019ba755ad use 53 bit hash fn to further avoid collisions trxcllnt Jun 6, 20199c3a865 ensure byteLength is calculated for offsets buffer trxcllnt Jun 6, 2019c0f7f7b fix a few minor formatting issues in arrow2csv trxcllnt Jun 7, 201905acad8 fix minor row serialization issues to be compatible with console.table() trxcllnt Jun 7, 20194d2a4f0 regenerate flatbuffer source from current format schemas trxcllnt Jun 7, 20198b0752f fix possible AsyncRandomAccessFile race condition retrieving filehand… trxcllnt Jun 7, 201907fa443 remove default dictionary hash function trxcllnt Jun 19, 20197998d2a add createIsValidFunction example docstring trxcllnt Jun 19, 2019 File filter Filter by extension Filter by extension .js (5) .json (4) .ts (117) All 3 file types selected Only manifest files Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace js/.vscode/launch.json launch.json js/bin/print-buffer-alignment.js print-buffer-alignment.js js/gulp/closure-task.js closure-task.js js/gulp/test-task.js test-task.js js/gulp/util.js util.js js/package-lock.json package-lock.json js/package.json package.json js/src/Arrow.dom.ts Arrow.dom.ts js/src/Arrow.node.ts Arrow.node.ts js/src/Arrow.ts Arrow.ts js/src/bin/arrow2csv.ts arrow2csv.ts js/src/builder.ts builder.ts js/src/builder/binary.ts binary.ts js/src/builder/bool.ts bool.ts js/src/builder/buffer.ts buffer.ts js/src/builder/date.ts date.ts js/src/builder/decimal.ts decimal.ts js/src/builder/dictionary.ts dictionary.ts js/src/builder/fixedsizebinary.ts fixedsizebinary.ts js/src/builder/fixedsizelist.ts fixedsizelist.ts js/src/builder/float.ts float.ts js/src/builder/index.ts index.ts js/src/builder/int.ts int.ts js/src/builder/interval.ts interval.ts js/src/builder/list.ts list.ts js/src/builder/map.ts map.ts js/src/builder/null.ts null.ts js/src/builder/run.ts run.ts js/src/builder/struct.ts struct.ts js/src/builder/time.ts time.ts js/src/builder/timestamp.ts timestamp.ts js/src/builder/union.ts union.ts js/src/builder/utf8.ts utf8.ts js/src/builder/valid.ts valid.ts js/src/column.ts column.ts js/src/compute/dataframe.ts dataframe.ts js/src/data.ts data.ts js/src/enum.ts enum.ts js/src/fb/File.ts File.ts js/src/fb/Message.ts Message.ts js/src/fb/Schema.ts Schema.ts js/src/interfaces.ts interfaces.ts js/src/io/adapters.ts adapters.ts js/src/io/file.ts file.ts js/src/io/node/builder.ts builder.ts js/src/io/node/iterable.ts iterable.ts js/src/io/node/reader.ts reader.ts js/src/io/node/writer.ts writer.ts js/src/io/stream.ts stream.ts js/src/io/whatwg/builder.ts builder.ts js/src/io/whatwg/iterable.ts iterable.ts js/src/io/whatwg/reader.ts reader.ts js/src/io/whatwg/writer.ts writer.ts js/src/ipc/metadata/json.ts json.ts js/src/ipc/metadata/message.ts message.ts js/src/ipc/writer.ts writer.ts js/src/recordbatch.ts recordbatch.ts js/src/schema.ts schema.ts js/src/table.ts table.ts js/src/type.ts type.ts js/src/util/args.ts args.ts js/src/util/bn.ts bn.ts js/src/util/buffer.ts buffer.ts js/src/util/compat.ts compat.ts js/src/util/fn.ts fn.ts js/src/util/int.ts int.ts js/src/util/utf8.ts utf8.ts js/src/util/vector.ts vector.ts js/src/vector.ts vector.ts js/src/vector/base.ts base.ts js/src/vector/binary.ts binary.ts js/src/vector/bool.ts bool.ts js/src/vector/chunked.ts chunked.ts js/src/vector/date.ts date.ts js/src/vector/decimal.ts decimal.ts js/src/vector/dictionary.ts dictionary.ts js/src/vector/fixedsizebinary.ts fixedsizebinary.ts js/src/vector/fixedsizelist.ts fixedsizelist.ts js/src/vector/float.ts float.ts js/src/vector/index.ts index.ts js/src/vector/int.ts int.ts js/src/vector/interval.ts interval.ts js/src/vector/list.ts list.ts js/src/vector/map.ts map.ts js/src/vector/null.ts null.ts js/src/vector/row.ts row.ts js/src/vector/struct.ts struct.ts js/src/vector/time.ts time.ts js/src/vector/timestamp.ts timestamp.ts js/src/vector/union.ts union.ts js/src/vector/utf8.ts utf8.ts js/src/visitor/builderctor.ts builderctor.ts js/src/visitor/bytewidth.ts bytewidth.ts js/src/visitor/get.ts get.ts js/src/visitor/indexof.ts indexof.ts js/src/visitor/iterator.ts iterator.ts js/src/visitor/jsontypeassembler.ts jsontypeassembler.ts js/src/visitor/jsonvectorassembler.ts jsonvectorassembler.ts js/src/visitor/set.ts set.ts js/src/visitor/toarray.ts toarray.ts js/src/visitor/typeassembler.ts typeassembler.ts js/src/visitor/typecomparator.ts typecomparator.ts js/src/visitor/typector.ts typector.ts js/src/visitor/vectorassembler.ts vectorassembler.ts js/src/visitor/vectorctor.ts vectorctor.ts js/src/visitor/vectorloader.ts vectorloader.ts js/test/Arrow.ts Arrow.ts js/test/generate-test-data.ts generate-test-data.ts js/test/inference/column.ts column.ts js/test/inference/nested.ts nested.ts js/test/jest-extensions.ts jest-extensions.ts js/test/unit/builders/builder-tests.ts builder-tests.ts js/test/unit/builders/date-tests.ts date-tests.ts js/test/unit/builders/dictionary-tests.ts dictionary-tests.ts js/test/unit/builders/int64-tests.ts int64-tests.ts js/test/unit/builders/primitive-tests.ts primitive-tests.ts js/test/unit/builders/uint64-tests.ts uint64-tests.ts js/test/unit/builders/utf8-tests.ts utf8-tests.ts js/test/unit/builders/utils.ts utils.ts js/test/unit/generated-data-tests.ts generated-data-tests.ts js/test/unit/ipc/writer/streams-dom-tests.ts streams-dom-tests.ts js/test/unit/ipc/writer/streams-node-tests.ts streams-node-tests.ts js/test/unit/vector/bool-vector-tests.ts bool-vector-tests.ts js/test/unit/vector/numeric-vector-tests.ts numeric-vector-tests.ts js/tsconfig/tsconfig.base.json tsconfig.base.json js/typedoc.js typedoc.js There are no files selected for viewing 11 changes: 10 additions & 1 deletion 11 js/.vscode/launch.json Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -37,8 +37,17 @@ // "TEST_MODULE": "umd" }, "args": [ // "-i", "-i", "test/unit/", // "test/unit/builders/", // "test/unit/builders/builder-tests.ts", // "test/unit/builders/int64-tests.ts", // "test/unit/builders/uint64-tests.ts", // "test/unit/builders/date-tests.ts", // "test/unit/builders/primitive-tests.ts", // "test/unit/builders/dictionary-tests.ts", // "test/unit/builders/utf8-tests.ts", // Uncomment any of these to run individual test suites // "test/unit/int-tests.ts", Expand Down 36 changes: 30 additions & 6 deletions 36 js/bin/print-buffer-alignment.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,29 +22,45 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; const { AsyncMessageReader } = require(`../index${extension}`); const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); let recordBatchIndex = 0, dictionaryBatchIndex = 0; let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; for await (let message of reader) { let bufferRegions = []; if (message.isSchema()) { schema = message.header(); continue; } else if (message.isRecordBatch()) { bufferRegions = message.header().buffers; const header = message.header(); bufferRegions = header.buffers; const body = await reader.readMessageBody(message.bodyLength); console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadRecordBatch(schema, header, body); console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, })}`); } else if (message.isDictionaryBatch()) { bufferRegions = message.header().data.buffers; const header = message.header(); bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); const body = await reader.readMessageBody(message.bodyLength); console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadDictionaryBatch(header.data, body, type); console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, dictionaryId: header.id, })}`); } bufferRegions.forEach(({ offset, length }, i) => { Expand All @@ -55,3 +71,11 @@ const { AsyncMessageReader } = require(`../index${extension}`); await reader.return(); })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields)); } function loadDictionaryBatch(header, body, dictionaryType) { return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType])); } 1 change: 0 additions & 1 deletion 1 js/gulp/closure-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ // formatting: `PRETTY_PRINT`, // debug: true, compilation_level: `ADVANCED`, allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/test-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.verbose && jestArgv.push(`--verbose`); argv.coverage ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`) ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`) : jestArgv.push(`-c`, `jest.config.js`, `-i`) const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/util.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) { env: { ...process.env, NODE_NO_WARNINGS: `1` } }; return asyncDone(() => child_process.spawn(`node`, args, opts)) .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; }); .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); } const logAndDie = (e) => { if (e) { process.exit(1); } }; Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /arrowPublic Notifications You must be signed in to change notification settings Fork 4.3k Star 17.1k Code Issues2.2k Pull requests401 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights ARROW-5115: [JS] Add Vector Builders and high-level stream primitives#4476 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Closed trxcllnt wants to merge 56 commits into apache:master from trxcllnt:js/data-builders Conversation 23 Commits 56 Checks 0 Files changed Uh oh! There was an error while loading. Please reload this page. Closed ARROW-5115: [JS] Add Vector Builders and high-level stream primitives #4476 Show file tree Hide file tree Changes from all commits Commits Show all changes 56 commits Select commit Hold shift + click to select a range 738fabb WIP streaming data builders trxcllnt Feb 1, 2019ff32cbb fix date builder tests trxcllnt Feb 3, 20199e5e3ea fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196ebaa36 WIP streaming data builders trxcllnt Feb 1, 20192f423ec fix date builder tests trxcllnt Feb 3, 2019c609266 fix bool builder, add primitive builder tests trxcllnt Feb 4, 20196b7a20e ensure 64-bit int builders use BN to check for nulls trxcllnt Feb 6, 20198a5c713 show a better error message if gulp fails trxcllnt Mar 20, 2019cf5f778 return signed or unsigned 53bit integers trxcllnt Mar 20, 20196e64c40 ensure builder allocates aligned buffers, null bitmaps initialized to… trxcllnt Mar 20, 20192b7b992 ensure union typeids buffer is an Int8Array trxcllnt Mar 20, 2019f7fe8c1 update builder buffer padding trxcllnt Apr 7, 20191398ebd don't clone Dictionary DataType instances in Schema assign to preserv… trxcllnt Apr 8, 2019fb2c9a2 cleanup trxcllnt Apr 9, 20192f36741 move stream methods to io folder trxcllnt Apr 15, 2019385fee3 fix typo trxcllnt Apr 15, 2019d193141 add helper method to return the stride for a datatype trxcllnt Apr 15, 201970377a1 add vectorname getter to chunked for completion trxcllnt Apr 15, 2019f8760cb enumerate each type key individually now that they're a real thing trxcllnt Apr 15, 20199ba9965 update row type inference sanity check trxcllnt Apr 15, 20194c2ef42 add the rest of the builder types trxcllnt Apr 15, 2019a307273 fix readable-stream detection in firefox trxcllnt Apr 16, 2019f651657 update typescript, ts-jest, jest trxcllnt Apr 16, 2019f708190 use safe BigInt64Array constructor trxcllnt Apr 17, 201915ac17c use Object.setPrototypeOf to improve bn performance trxcllnt May 11, 20190990344 add Builder throughDOM and throughNode transform streams trxcllnt May 17, 20192b7a63c update test types trxcllnt May 22, 201979bfd46 fix node and dom builder streams trxcllnt May 22, 201926387c7 update typescript, finish streaming builders, add comprehensive build… trxcllnt May 27, 2019fd46163 fix nan checks trxcllnt May 27, 2019d032e2d update dependencies trxcllnt May 27, 201986a6cd0 update closure compiler dependency trxcllnt May 27, 2019c587c33 add BufferBuilders, clean up public Builder API trxcllnt May 29, 20190209e9d update to typescript 3.5 trxcllnt May 30, 201984beb68 remove some property getters, clean up trxcllnt May 30, 2019dddba1a ensure variablewidthbuilder supports random insertion trxcllnt May 31, 2019893c74f ensure ListBuilder supports random insertion trxcllnt Jun 1, 2019c2673ac add initial builder jsdoc trxcllnt Jun 4, 20195b990c3 Clean up typedoc output, update typedoc to master to use typescript@3… trxcllnt Jun 5, 2019b2556b7 add docstring for _InternalEmptyPlaceholderRecordBatch trxcllnt Jun 5, 20194b5375e remove unnecessary jsdoc trxcllnt Jun 5, 20195d4b0be ensure bitmapbufferbuilder increments and decrements _popCount approp… trxcllnt Jun 5, 2019e974444 fix lint trxcllnt Jun 5, 20195aea4f9 Add more specific Int64 and Uint64 Builder tests trxcllnt Jun 5, 20193adf555 adds or updates most of the high-level Vector.from() methods to use t… trxcllnt Jun 6, 2019f250845 use a better default dictionary builder hash function trxcllnt Jun 6, 201913b51db print more details about each message trxcllnt Jun 6, 20198659323 add test for builder iterable byte queueing strategy trxcllnt Jun 6, 2019ba755ad use 53 bit hash fn to further avoid collisions trxcllnt Jun 6, 20199c3a865 ensure byteLength is calculated for offsets buffer trxcllnt Jun 6, 2019c0f7f7b fix a few minor formatting issues in arrow2csv trxcllnt Jun 7, 201905acad8 fix minor row serialization issues to be compatible with console.table() trxcllnt Jun 7, 20194d2a4f0 regenerate flatbuffer source from current format schemas trxcllnt Jun 7, 20198b0752f fix possible AsyncRandomAccessFile race condition retrieving filehand… trxcllnt Jun 7, 201907fa443 remove default dictionary hash function trxcllnt Jun 19, 20197998d2a add createIsValidFunction example docstring trxcllnt Jun 19, 2019 File filter Filter by extension Filter by extension .js (5) .json (4) .ts (117) All 3 file types selected Only manifest files Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace js/.vscode/launch.json launch.json js/bin/print-buffer-alignment.js print-buffer-alignment.js js/gulp/closure-task.js closure-task.js js/gulp/test-task.js test-task.js js/gulp/util.js util.js js/package-lock.json package-lock.json js/package.json package.json js/src/Arrow.dom.ts Arrow.dom.ts js/src/Arrow.node.ts Arrow.node.ts js/src/Arrow.ts Arrow.ts js/src/bin/arrow2csv.ts arrow2csv.ts js/src/builder.ts builder.ts js/src/builder/binary.ts binary.ts js/src/builder/bool.ts bool.ts js/src/builder/buffer.ts buffer.ts js/src/builder/date.ts date.ts js/src/builder/decimal.ts decimal.ts js/src/builder/dictionary.ts dictionary.ts js/src/builder/fixedsizebinary.ts fixedsizebinary.ts js/src/builder/fixedsizelist.ts fixedsizelist.ts js/src/builder/float.ts float.ts js/src/builder/index.ts index.ts js/src/builder/int.ts int.ts js/src/builder/interval.ts interval.ts js/src/builder/list.ts list.ts js/src/builder/map.ts map.ts js/src/builder/null.ts null.ts js/src/builder/run.ts run.ts js/src/builder/struct.ts struct.ts js/src/builder/time.ts time.ts js/src/builder/timestamp.ts timestamp.ts js/src/builder/union.ts union.ts js/src/builder/utf8.ts utf8.ts js/src/builder/valid.ts valid.ts js/src/column.ts column.ts js/src/compute/dataframe.ts dataframe.ts js/src/data.ts data.ts js/src/enum.ts enum.ts js/src/fb/File.ts File.ts js/src/fb/Message.ts Message.ts js/src/fb/Schema.ts Schema.ts js/src/interfaces.ts interfaces.ts js/src/io/adapters.ts adapters.ts js/src/io/file.ts file.ts js/src/io/node/builder.ts builder.ts js/src/io/node/iterable.ts iterable.ts js/src/io/node/reader.ts reader.ts js/src/io/node/writer.ts writer.ts js/src/io/stream.ts stream.ts js/src/io/whatwg/builder.ts builder.ts js/src/io/whatwg/iterable.ts iterable.ts js/src/io/whatwg/reader.ts reader.ts js/src/io/whatwg/writer.ts writer.ts js/src/ipc/metadata/json.ts json.ts js/src/ipc/metadata/message.ts message.ts js/src/ipc/writer.ts writer.ts js/src/recordbatch.ts recordbatch.ts js/src/schema.ts schema.ts js/src/table.ts table.ts js/src/type.ts type.ts js/src/util/args.ts args.ts js/src/util/bn.ts bn.ts js/src/util/buffer.ts buffer.ts js/src/util/compat.ts compat.ts js/src/util/fn.ts fn.ts js/src/util/int.ts int.ts js/src/util/utf8.ts utf8.ts js/src/util/vector.ts vector.ts js/src/vector.ts vector.ts js/src/vector/base.ts base.ts js/src/vector/binary.ts binary.ts js/src/vector/bool.ts bool.ts js/src/vector/chunked.ts chunked.ts js/src/vector/date.ts date.ts js/src/vector/decimal.ts decimal.ts js/src/vector/dictionary.ts dictionary.ts js/src/vector/fixedsizebinary.ts fixedsizebinary.ts js/src/vector/fixedsizelist.ts fixedsizelist.ts js/src/vector/float.ts float.ts js/src/vector/index.ts index.ts js/src/vector/int.ts int.ts js/src/vector/interval.ts interval.ts js/src/vector/list.ts list.ts js/src/vector/map.ts map.ts js/src/vector/null.ts null.ts js/src/vector/row.ts row.ts js/src/vector/struct.ts struct.ts js/src/vector/time.ts time.ts js/src/vector/timestamp.ts timestamp.ts js/src/vector/union.ts union.ts js/src/vector/utf8.ts utf8.ts js/src/visitor/builderctor.ts builderctor.ts js/src/visitor/bytewidth.ts bytewidth.ts js/src/visitor/get.ts get.ts js/src/visitor/indexof.ts indexof.ts js/src/visitor/iterator.ts iterator.ts js/src/visitor/jsontypeassembler.ts jsontypeassembler.ts js/src/visitor/jsonvectorassembler.ts jsonvectorassembler.ts js/src/visitor/set.ts set.ts js/src/visitor/toarray.ts toarray.ts js/src/visitor/typeassembler.ts typeassembler.ts js/src/visitor/typecomparator.ts typecomparator.ts js/src/visitor/typector.ts typector.ts js/src/visitor/vectorassembler.ts vectorassembler.ts js/src/visitor/vectorctor.ts vectorctor.ts js/src/visitor/vectorloader.ts vectorloader.ts js/test/Arrow.ts Arrow.ts js/test/generate-test-data.ts generate-test-data.ts js/test/inference/column.ts column.ts js/test/inference/nested.ts nested.ts js/test/jest-extensions.ts jest-extensions.ts js/test/unit/builders/builder-tests.ts builder-tests.ts js/test/unit/builders/date-tests.ts date-tests.ts js/test/unit/builders/dictionary-tests.ts dictionary-tests.ts js/test/unit/builders/int64-tests.ts int64-tests.ts js/test/unit/builders/primitive-tests.ts primitive-tests.ts js/test/unit/builders/uint64-tests.ts uint64-tests.ts js/test/unit/builders/utf8-tests.ts utf8-tests.ts js/test/unit/builders/utils.ts utils.ts js/test/unit/generated-data-tests.ts generated-data-tests.ts js/test/unit/ipc/writer/streams-dom-tests.ts streams-dom-tests.ts js/test/unit/ipc/writer/streams-node-tests.ts streams-node-tests.ts js/test/unit/vector/bool-vector-tests.ts bool-vector-tests.ts js/test/unit/vector/numeric-vector-tests.ts numeric-vector-tests.ts js/tsconfig/tsconfig.base.json tsconfig.base.json js/typedoc.js typedoc.js There are no files selected for viewing 11 changes: 10 additions & 1 deletion 11 js/.vscode/launch.json Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -37,8 +37,17 @@ // "TEST_MODULE": "umd" }, "args": [ // "-i", "-i", "test/unit/", // "test/unit/builders/", // "test/unit/builders/builder-tests.ts", // "test/unit/builders/int64-tests.ts", // "test/unit/builders/uint64-tests.ts", // "test/unit/builders/date-tests.ts", // "test/unit/builders/primitive-tests.ts", // "test/unit/builders/dictionary-tests.ts", // "test/unit/builders/utf8-tests.ts", // Uncomment any of these to run individual test suites // "test/unit/int-tests.ts", Expand Down 36 changes: 30 additions & 6 deletions 36 js/bin/print-buffer-alignment.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,29 +22,45 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; const { AsyncMessageReader } = require(`../index${extension}`); const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); let recordBatchIndex = 0, dictionaryBatchIndex = 0; let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; for await (let message of reader) { let bufferRegions = []; if (message.isSchema()) { schema = message.header(); continue; } else if (message.isRecordBatch()) { bufferRegions = message.header().buffers; const header = message.header(); bufferRegions = header.buffers; const body = await reader.readMessageBody(message.bodyLength); console.log(`record batch ${++recordBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadRecordBatch(schema, header, body); console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, })}`); } else if (message.isDictionaryBatch()) { bufferRegions = message.header().data.buffers; const header = message.header(); bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); const body = await reader.readMessageBody(message.bodyLength); console.log(`dictionary batch ${++dictionaryBatchIndex}, byteOffset ${body.byteOffset}`); const recordBatch = loadDictionaryBatch(header.data, body, type); console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ offset: body.byteOffset, length: body.byteLength, numRows: recordBatch.length, dictionaryId: header.id, })}`); } bufferRegions.forEach(({ offset, length }, i) => { Expand All @@ -55,3 +71,11 @@ const { AsyncMessageReader } = require(`../index${extension}`); await reader.return(); })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields)); } function loadDictionaryBatch(header, body, dictionaryType) { return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType])); } 1 change: 0 additions & 1 deletion 1 js/gulp/closure-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -99,7 +99,6 @@ const createClosureArgs = (entry_point, externs) => ({ // formatting: `PRETTY_PRINT`, // debug: true, compilation_level: `ADVANCED`, allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/test-task.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -32,7 +32,7 @@ const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.verbose && jestArgv.push(`--verbose`); argv.coverage ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`) ? jestArgv.push(`-c`, `jest.coverage.config.js`, `--coverage`, `-i`) : jestArgv.push(`-c`, `jest.config.js`, `-i`) const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); Expand Down 2 changes: 1 addition & 1 deletion 2 js/gulp/util.js Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -129,7 +129,7 @@ function spawnGulpCommandInChildProcess(command, target, format) { env: { ...process.env, NODE_NO_WARNINGS: `1` } }; return asyncDone(() => child_process.spawn(`node`, args, opts)) .catch((e) => { throw { message: `${command}:${taskName(target, format)}` }; }); .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); } const logAndDie = (e) => { if (e) { process.exit(1); } }; Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time.