ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
trxcllnt wants to merge 1 commit into from
Closed

ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder #4502

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions js/bin/print-buffer-alignment.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}
16 changes: 2 additions & 14 deletions js/src/builder.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
* @nocollapse
*/
public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
const build = throughIterable(options);
if (!DataType.isDictionary(options.type)) {
return build;
}
return function*(source: Iterable<T['TValue'] | TNull>) {
const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
};
return throughIterable(options);
}

/**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
* @nocollapse
*/
public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
const build = throughAsyncIterable(options);
if (!DataType.isDictionary(options.type)) {
return build;
}
return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
};
return throughAsyncIterable(options);
}

/**
Expand Down
36 changes: 25 additions & 11 deletions js/src/builder/dictionary.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

protected _codes = Object.create(null);
protected _dictionaryOffset: number;
protected _dictionary?: Vector<T['dictionary']>;
protected _keysToIndices: { [key: string]: number };
public readonly indices: IntBuilder<T['indices']>;
public readonly dictionary: Builder<T['dictionary']>;

constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
super({ type });
super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
this._nulls = <any> null;
this._dictionaryOffset = 0;
this._keysToIndices = Object.create(null);
this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
public get values() { return this.indices.values; }
public get nullCount() { return this.indices.nullCount; }
public get nullBitmap() { return this.indices.nullBitmap; }
public get byteLength() { return this.indices.byteLength; }
public get reservedLength() { return this.indices.reservedLength; }
public get reservedByteLength() { return this.indices.reservedByteLength; }
public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
public setValid(index: number, valid: boolean) {
const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
return valid;
}
public setValue(index: number, value: T['TValue']) {
let keysToCodesMap = this._codes;
let keysToIndices = this._keysToIndices;
let key = this.valueToKey(value);
let idx = keysToCodesMap[key];
let idx = keysToIndices[key];
if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
}
return this.indices.setValue(index, idx);
}
public flush() {
const chunk = this.indices.flush().clone(this.type);
const type = this.type;
const prev = this._dictionary;
const curr = this.dictionary.toVector();
const data = this.indices.flush().clone(type);
data.dictionary = prev ? prev.concat(curr) : curr;
this.finished || (this._dictionaryOffset += curr.length);
this._dictionary = data.dictionary as Vector<T['dictionary']>;
this.clear();
return chunk;
return data;
}
public finish() {
this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
this.indices.finish();
this.dictionary.finish();
this._dictionaryOffset = 0;
this._keysToIndices = Object.create(null);
return super.finish();
}
public clear() {
this.indices.clear();
this.dictionary.clear();
return super.clear();
}
public valueToKey(val: any): string | number {
Expand Down
2 changes: 1 addition & 1 deletion js/src/column.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

if (typeof field === 'string') {
const type = chunks[0].data.type;
field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
field = new Field(field, type, true);
} else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
field = field.clone({ nullable: true });
}
Expand Down
19 changes: 13 additions & 6 deletions js/src/data.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
public readonly offset: number;
public readonly stride: number;
public readonly childData: Data[];

/**
* The dictionary for this Vector, if any. Only used for Dictionary type.
*/
public dictionary?: Vector;

public readonly values: Buffers<T>[BufferType.DATA];
// @ts-ignore
public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
return nullCount;
}

constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
this.type = type;
this.dictionary = dictionary;
this.offset = Math.floor(Math.max(offset || 0, 0));
this.length = Math.floor(Math.max(length || 0, 0));
this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
}

public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
return new Data(type, offset, length, nullCount, buffers, childData);
return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
}

public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
// Convenience methods for creating Data instances for each of the Arrow Vector types
//
/** @nocollapse */
public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
switch (type.typeId) {
case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
}
/** @nocollapse */
public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
}
/** @nocollapse */
public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down
4 changes: 2 additions & 2 deletions js/src/interfaces.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
T extends VectorType ? VectorCtorType<T> :
T extends VectorType ? VectorCtorType<T> :
T extends Type ? VectorCtorType<VectorType<T>> :
T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
T extends DataType ? ConstructorType<T> :
T extends VectorType ? ConstructorType<T['type']> :
T extends VectorType ? ConstructorType<T['type']> :
T extends Type ? ConstructorType<TypeToDataType<T>> :
never
;
Expand Down
15 changes: 0 additions & 15 deletions js/src/io/node/builder.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

const isDictionary = DataType.isDictionary(builder.type);
const { queueingStrategy = 'count', autoDestroy = true } = options;
const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
this._builder = builder;
this._desiredSize = highWaterMark;
this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

if (isDictionary) {
let chunks: any[] = [];
this.push = (chunk: any, _?: string) => {
if (chunk !== null) {
chunks.push(chunk);
return true;
}
const chunks_ = chunks;
chunks = [];
chunks_.forEach((x) => super.push(x));
return super.push(null) && false;
};
}
}
_read(size: number) {
this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down
16 changes: 0 additions & 16 deletions js/src/io/whatwg/builder.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
'highWaterMark': writableHighWaterMark,
'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
});

if (DataType.isDictionary(builderOptions.type)) {
let chunks: any[] = [];
this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
this._bufferedSize = 0;
if (chunk !== null) {
chunks.push(chunk);
} else {
const chunks_ = chunks;
chunks = [];
chunks_.forEach((x) => controller.enqueue(x));
controller.close();
this._controller = null;
}
};
}
}

private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down
29 changes: 13 additions & 16 deletions js/src/ipc/metadata/json.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
return new Schema(
schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
schemaFieldsFromJSON(_schema, dictionaries),
customMetadataFromJSON(_schema['customMetadata']),
dictionaries, dictionaryFields
dictionaries
);
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

let id: number;
let keys: TKeys | null;
let field: Field | void;
let dictMeta: any;
let type: DataType<any>;
let dictType: Dictionary;
let dictField: Field<Dictionary>;

// If no dictionary encoding
if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
if (!dictionaries || !(dictMeta = _field['dictionary'])) {
type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
}
// tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
else if (!dictionaries.has(id = dictMeta['id'])) {
// a dictionary index defaults to signed 32 bit int if unspecified
keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
dictionaryFields.set(id, [field = dictField]);
field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
}
// If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
// data type and wrap in a new Dictionary type and field.
else {
// a dictionary index defaults to signed 32 bit int if unspecified
keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
dictionaryFields.get(id)!.push(field = dictField);
field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
}
return field || null;
}
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

trxcllnt
wants to merge
1
commit into

from

 Closed


ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
#4502
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 2 additions & 2 deletions
js/bin/print-buffer-alignment.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}

16 changes: 2 additions & 14 deletions
js/src/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return function*(source: Iterable<T['TValue'] | TNull>) {
 const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughIterable(options);
 }

 /**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughAsyncIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
 const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughAsyncIterable(options);
 }

 /**
Expand Down

36 changes: 25 additions & 11 deletions
js/src/builder/dictionary.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

 protected _codes = Object.create(null);
 protected _dictionaryOffset: number;
 protected _dictionary?: Vector<T['dictionary']>;
 protected _keysToIndices: { [key: string]: number };
 public readonly indices: IntBuilder<T['indices']>;
 public readonly dictionary: Builder<T['dictionary']>;

 constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
 super({ type });
 super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
 this._nulls = <any> null;
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
 this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
 if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 public get values() { return this.indices.values; }
 public get nullCount() { return this.indices.nullCount; }
 public get nullBitmap() { return this.indices.nullBitmap; }
 public get byteLength() { return this.indices.byteLength; }
 public get reservedLength() { return this.indices.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength; }
 public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
 public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
 public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
 public setValid(index: number, valid: boolean) {
 const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 return valid;
 }
 public setValue(index: number, value: T['TValue']) {
 let keysToCodesMap = this._codes;
 let keysToIndices = this._keysToIndices;
 let key = this.valueToKey(value);
 let idx = keysToCodesMap[key];
 let idx = keysToIndices[key];
 if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
 }
 return this.indices.setValue(index, idx);
 }
 public flush() {
 const chunk = this.indices.flush().clone(this.type);
 const type = this.type;
 const prev = this._dictionary;
 const curr = this.dictionary.toVector();
 const data = this.indices.flush().clone(type);
 data.dictionary = prev ? prev.concat(curr) : curr;
 this.finished || (this._dictionaryOffset += curr.length);
 this._dictionary = data.dictionary as Vector<T['dictionary']>;
 this.clear();
 return chunk;
 return data;
 }
 public finish() {
 this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
 this.indices.finish();
 this.dictionary.finish();
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 return super.finish();
 }
 public clear() {
 this.indices.clear();
 this.dictionary.clear();
 return super.clear();
 }
 public valueToKey(val: any): string | number {
Expand Down

2 changes: 1 addition & 1 deletion
js/src/column.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

 if (typeof field === 'string') {
 const type = chunks[0].data.type;
 field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
 field = new Field(field, type, true);
 } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
 field = field.clone({ nullable: true });
 }
Expand Down

19 changes: 13 additions & 6 deletions
js/src/data.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
 public readonly offset: number;
 public readonly stride: number;
 public readonly childData: Data[];

 /**
 * The dictionary for this Vector, if any. Only used for Dictionary type.
 */
 public dictionary?: Vector;

 public readonly values: Buffers<T>[BufferType.DATA];
 // @ts-ignore
 public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
 return nullCount;
 }

 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
 this.type = type;
 this.dictionary = dictionary;
 this.offset = Math.floor(Math.max(offset || 0, 0));
 this.length = Math.floor(Math.max(length || 0, 0));
 this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
 }

 public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
 return new Data(type, offset, length, nullCount, buffers, childData);
 return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
 }

 public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
 // Convenience methods for creating Data instances for each of the Arrow Vector types
 //
 /** @nocollapse */
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
 if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
 switch (type.typeId) {
 case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
 case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
 case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
 }
 /** @nocollapse */
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
 }
 /** @nocollapse */
 public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down

4 changes: 2 additions & 2 deletions
js/src/interfaces.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
 T extends VectorType ? VectorCtorType<T> :
 T extends VectorType ? VectorCtorType<T> :
 T extends Type ? VectorCtorType<VectorType<T>> :
 T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
 VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
 T extends DataType ? ConstructorType<T> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends Type ? ConstructorType<TypeToDataType<T>> :
 never
 ;
Expand Down

15 changes: 0 additions & 15 deletions
js/src/io/node/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

 constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

 const isDictionary = DataType.isDictionary(builder.type);
 const { queueingStrategy = 'count', autoDestroy = true } = options;
 const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
 this._builder = builder;
 this._desiredSize = highWaterMark;
 this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

 if (isDictionary) {
 let chunks: any[] = [];
 this.push = (chunk: any, _?: string) => {
 if (chunk !== null) {
 chunks.push(chunk);
 return true;
 }
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => super.push(x));
 return super.push(null) && false;
 };
 }
 }
 _read(size: number) {
 this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down

16 changes: 0 additions & 16 deletions
js/src/io/whatwg/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
 'highWaterMark': writableHighWaterMark,
 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
 });

 if (DataType.isDictionary(builderOptions.type)) {
 let chunks: any[] = [];
 this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
 this._bufferedSize = 0;
 if (chunk !== null) {
 chunks.push(chunk);
 } else {
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => controller.enqueue(x));
 controller.close();
 this._controller = null;
 }
 };
 }
 }

 private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down

29 changes: 13 additions & 16 deletions
js/src/ipc/metadata/json.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
 return new Schema(
 schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
 schemaFieldsFromJSON(_schema, dictionaries),
 customMetadataFromJSON(_schema['customMetadata']),
 dictionaries, dictionaryFields
 dictionaries
 );
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

 let id: number;
 let keys: TKeys | null;
 let field: Field | void;
 let dictMeta: any;
 let type: DataType<any>;
 let dictType: Dictionary;
 let dictField: Field<Dictionary>;

 // If no dictionary encoding
 if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
 if (!dictionaries || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
 field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
 else if (!dictionaries.has(id = dictMeta['id'])) {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
 dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.set(id, [field = dictField]);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
 // data type and wrap in a new Dictionary type and field.
 else {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.get(id)!.push(field = dictField);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 return field || null;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

trxcllnt
wants to merge
1
commit into

from

 Closed


ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
#4502
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 2 additions & 2 deletions
js/bin/print-buffer-alignment.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}

16 changes: 2 additions & 14 deletions
js/src/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return function*(source: Iterable<T['TValue'] | TNull>) {
 const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughIterable(options);
 }

 /**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughAsyncIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
 const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughAsyncIterable(options);
 }

 /**
Expand Down

36 changes: 25 additions & 11 deletions
js/src/builder/dictionary.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

 protected _codes = Object.create(null);
 protected _dictionaryOffset: number;
 protected _dictionary?: Vector<T['dictionary']>;
 protected _keysToIndices: { [key: string]: number };
 public readonly indices: IntBuilder<T['indices']>;
 public readonly dictionary: Builder<T['dictionary']>;

 constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
 super({ type });
 super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
 this._nulls = <any> null;
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
 this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
 if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 public get values() { return this.indices.values; }
 public get nullCount() { return this.indices.nullCount; }
 public get nullBitmap() { return this.indices.nullBitmap; }
 public get byteLength() { return this.indices.byteLength; }
 public get reservedLength() { return this.indices.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength; }
 public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
 public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
 public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
 public setValid(index: number, valid: boolean) {
 const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 return valid;
 }
 public setValue(index: number, value: T['TValue']) {
 let keysToCodesMap = this._codes;
 let keysToIndices = this._keysToIndices;
 let key = this.valueToKey(value);
 let idx = keysToCodesMap[key];
 let idx = keysToIndices[key];
 if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
 }
 return this.indices.setValue(index, idx);
 }
 public flush() {
 const chunk = this.indices.flush().clone(this.type);
 const type = this.type;
 const prev = this._dictionary;
 const curr = this.dictionary.toVector();
 const data = this.indices.flush().clone(type);
 data.dictionary = prev ? prev.concat(curr) : curr;
 this.finished || (this._dictionaryOffset += curr.length);
 this._dictionary = data.dictionary as Vector<T['dictionary']>;
 this.clear();
 return chunk;
 return data;
 }
 public finish() {
 this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
 this.indices.finish();
 this.dictionary.finish();
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 return super.finish();
 }
 public clear() {
 this.indices.clear();
 this.dictionary.clear();
 return super.clear();
 }
 public valueToKey(val: any): string | number {
Expand Down

2 changes: 1 addition & 1 deletion
js/src/column.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

 if (typeof field === 'string') {
 const type = chunks[0].data.type;
 field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
 field = new Field(field, type, true);
 } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
 field = field.clone({ nullable: true });
 }
Expand Down

19 changes: 13 additions & 6 deletions
js/src/data.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
 public readonly offset: number;
 public readonly stride: number;
 public readonly childData: Data[];

 /**
 * The dictionary for this Vector, if any. Only used for Dictionary type.
 */
 public dictionary?: Vector;

 public readonly values: Buffers<T>[BufferType.DATA];
 // @ts-ignore
 public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
 return nullCount;
 }

 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
 this.type = type;
 this.dictionary = dictionary;
 this.offset = Math.floor(Math.max(offset || 0, 0));
 this.length = Math.floor(Math.max(length || 0, 0));
 this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
 }

 public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
 return new Data(type, offset, length, nullCount, buffers, childData);
 return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
 }

 public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
 // Convenience methods for creating Data instances for each of the Arrow Vector types
 //
 /** @nocollapse */
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
 if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
 switch (type.typeId) {
 case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
 case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
 case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
 }
 /** @nocollapse */
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
 }
 /** @nocollapse */
 public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down

4 changes: 2 additions & 2 deletions
js/src/interfaces.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
 T extends VectorType ? VectorCtorType<T> :
 T extends VectorType ? VectorCtorType<T> :
 T extends Type ? VectorCtorType<VectorType<T>> :
 T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
 VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
 T extends DataType ? ConstructorType<T> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends Type ? ConstructorType<TypeToDataType<T>> :
 never
 ;
Expand Down

15 changes: 0 additions & 15 deletions
js/src/io/node/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

 constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

 const isDictionary = DataType.isDictionary(builder.type);
 const { queueingStrategy = 'count', autoDestroy = true } = options;
 const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
 this._builder = builder;
 this._desiredSize = highWaterMark;
 this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

 if (isDictionary) {
 let chunks: any[] = [];
 this.push = (chunk: any, _?: string) => {
 if (chunk !== null) {
 chunks.push(chunk);
 return true;
 }
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => super.push(x));
 return super.push(null) && false;
 };
 }
 }
 _read(size: number) {
 this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down

16 changes: 0 additions & 16 deletions
js/src/io/whatwg/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
 'highWaterMark': writableHighWaterMark,
 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
 });

 if (DataType.isDictionary(builderOptions.type)) {
 let chunks: any[] = [];
 this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
 this._bufferedSize = 0;
 if (chunk !== null) {
 chunks.push(chunk);
 } else {
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => controller.enqueue(x));
 controller.close();
 this._controller = null;
 }
 };
 }
 }

 private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down

29 changes: 13 additions & 16 deletions
js/src/ipc/metadata/json.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
 return new Schema(
 schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
 schemaFieldsFromJSON(_schema, dictionaries),
 customMetadataFromJSON(_schema['customMetadata']),
 dictionaries, dictionaryFields
 dictionaries
 );
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

 let id: number;
 let keys: TKeys | null;
 let field: Field | void;
 let dictMeta: any;
 let type: DataType<any>;
 let dictType: Dictionary;
 let dictField: Field<Dictionary>;

 // If no dictionary encoding
 if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
 if (!dictionaries || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
 field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
 else if (!dictionaries.has(id = dictMeta['id'])) {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
 dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.set(id, [field = dictField]);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
 // data type and wrap in a new Dictionary type and field.
 else {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.get(id)!.push(field = dictField);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 return field || null;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

trxcllnt
wants to merge
1
commit into

from

 Closed


ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
#4502
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 2 additions & 2 deletions
js/bin/print-buffer-alignment.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}

16 changes: 2 additions & 14 deletions
js/src/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return function*(source: Iterable<T['TValue'] | TNull>) {
 const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughIterable(options);
 }

 /**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughAsyncIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
 const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughAsyncIterable(options);
 }

 /**
Expand Down

36 changes: 25 additions & 11 deletions
js/src/builder/dictionary.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

 protected _codes = Object.create(null);
 protected _dictionaryOffset: number;
 protected _dictionary?: Vector<T['dictionary']>;
 protected _keysToIndices: { [key: string]: number };
 public readonly indices: IntBuilder<T['indices']>;
 public readonly dictionary: Builder<T['dictionary']>;

 constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
 super({ type });
 super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
 this._nulls = <any> null;
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
 this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
 if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 public get values() { return this.indices.values; }
 public get nullCount() { return this.indices.nullCount; }
 public get nullBitmap() { return this.indices.nullBitmap; }
 public get byteLength() { return this.indices.byteLength; }
 public get reservedLength() { return this.indices.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength; }
 public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
 public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
 public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
 public setValid(index: number, valid: boolean) {
 const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 return valid;
 }
 public setValue(index: number, value: T['TValue']) {
 let keysToCodesMap = this._codes;
 let keysToIndices = this._keysToIndices;
 let key = this.valueToKey(value);
 let idx = keysToCodesMap[key];
 let idx = keysToIndices[key];
 if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
 }
 return this.indices.setValue(index, idx);
 }
 public flush() {
 const chunk = this.indices.flush().clone(this.type);
 const type = this.type;
 const prev = this._dictionary;
 const curr = this.dictionary.toVector();
 const data = this.indices.flush().clone(type);
 data.dictionary = prev ? prev.concat(curr) : curr;
 this.finished || (this._dictionaryOffset += curr.length);
 this._dictionary = data.dictionary as Vector<T['dictionary']>;
 this.clear();
 return chunk;
 return data;
 }
 public finish() {
 this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
 this.indices.finish();
 this.dictionary.finish();
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 return super.finish();
 }
 public clear() {
 this.indices.clear();
 this.dictionary.clear();
 return super.clear();
 }
 public valueToKey(val: any): string | number {
Expand Down

2 changes: 1 addition & 1 deletion
js/src/column.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

 if (typeof field === 'string') {
 const type = chunks[0].data.type;
 field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
 field = new Field(field, type, true);
 } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
 field = field.clone({ nullable: true });
 }
Expand Down

19 changes: 13 additions & 6 deletions
js/src/data.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
 public readonly offset: number;
 public readonly stride: number;
 public readonly childData: Data[];

 /**
 * The dictionary for this Vector, if any. Only used for Dictionary type.
 */
 public dictionary?: Vector;

 public readonly values: Buffers<T>[BufferType.DATA];
 // @ts-ignore
 public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
 return nullCount;
 }

 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
 this.type = type;
 this.dictionary = dictionary;
 this.offset = Math.floor(Math.max(offset || 0, 0));
 this.length = Math.floor(Math.max(length || 0, 0));
 this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
 }

 public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
 return new Data(type, offset, length, nullCount, buffers, childData);
 return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
 }

 public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
 // Convenience methods for creating Data instances for each of the Arrow Vector types
 //
 /** @nocollapse */
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
 if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
 switch (type.typeId) {
 case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
 case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
 case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
 }
 /** @nocollapse */
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
 }
 /** @nocollapse */
 public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down

4 changes: 2 additions & 2 deletions
js/src/interfaces.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
 T extends VectorType ? VectorCtorType<T> :
 T extends VectorType ? VectorCtorType<T> :
 T extends Type ? VectorCtorType<VectorType<T>> :
 T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
 VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
 T extends DataType ? ConstructorType<T> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends Type ? ConstructorType<TypeToDataType<T>> :
 never
 ;
Expand Down

15 changes: 0 additions & 15 deletions
js/src/io/node/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

 constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

 const isDictionary = DataType.isDictionary(builder.type);
 const { queueingStrategy = 'count', autoDestroy = true } = options;
 const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
 this._builder = builder;
 this._desiredSize = highWaterMark;
 this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

 if (isDictionary) {
 let chunks: any[] = [];
 this.push = (chunk: any, _?: string) => {
 if (chunk !== null) {
 chunks.push(chunk);
 return true;
 }
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => super.push(x));
 return super.push(null) && false;
 };
 }
 }
 _read(size: number) {
 this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down

16 changes: 0 additions & 16 deletions
js/src/io/whatwg/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
 'highWaterMark': writableHighWaterMark,
 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
 });

 if (DataType.isDictionary(builderOptions.type)) {
 let chunks: any[] = [];
 this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
 this._bufferedSize = 0;
 if (chunk !== null) {
 chunks.push(chunk);
 } else {
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => controller.enqueue(x));
 controller.close();
 this._controller = null;
 }
 };
 }
 }

 private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down

29 changes: 13 additions & 16 deletions
js/src/ipc/metadata/json.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
 return new Schema(
 schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
 schemaFieldsFromJSON(_schema, dictionaries),
 customMetadataFromJSON(_schema['customMetadata']),
 dictionaries, dictionaryFields
 dictionaries
 );
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

 let id: number;
 let keys: TKeys | null;
 let field: Field | void;
 let dictMeta: any;
 let type: DataType<any>;
 let dictType: Dictionary;
 let dictField: Field<Dictionary>;

 // If no dictionary encoding
 if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
 if (!dictionaries || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
 field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
 else if (!dictionaries.has(id = dictMeta['id'])) {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
 dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.set(id, [field = dictField]);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
 // data type and wrap in a new Dictionary type and field.
 else {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.get(id)!.push(field = dictField);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 return field || null;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

trxcllnt
wants to merge
1
commit into

from

 Closed


ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
#4502
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 2 additions & 2 deletions
js/bin/print-buffer-alignment.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}

16 changes: 2 additions & 14 deletions
js/src/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return function*(source: Iterable<T['TValue'] | TNull>) {
 const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughIterable(options);
 }

 /**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughAsyncIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
 const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughAsyncIterable(options);
 }

 /**
Expand Down

36 changes: 25 additions & 11 deletions
js/src/builder/dictionary.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

 protected _codes = Object.create(null);
 protected _dictionaryOffset: number;
 protected _dictionary?: Vector<T['dictionary']>;
 protected _keysToIndices: { [key: string]: number };
 public readonly indices: IntBuilder<T['indices']>;
 public readonly dictionary: Builder<T['dictionary']>;

 constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
 super({ type });
 super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
 this._nulls = <any> null;
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
 this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
 if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 public get values() { return this.indices.values; }
 public get nullCount() { return this.indices.nullCount; }
 public get nullBitmap() { return this.indices.nullBitmap; }
 public get byteLength() { return this.indices.byteLength; }
 public get reservedLength() { return this.indices.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength; }
 public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
 public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
 public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
 public setValid(index: number, valid: boolean) {
 const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 return valid;
 }
 public setValue(index: number, value: T['TValue']) {
 let keysToCodesMap = this._codes;
 let keysToIndices = this._keysToIndices;
 let key = this.valueToKey(value);
 let idx = keysToCodesMap[key];
 let idx = keysToIndices[key];
 if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
 }
 return this.indices.setValue(index, idx);
 }
 public flush() {
 const chunk = this.indices.flush().clone(this.type);
 const type = this.type;
 const prev = this._dictionary;
 const curr = this.dictionary.toVector();
 const data = this.indices.flush().clone(type);
 data.dictionary = prev ? prev.concat(curr) : curr;
 this.finished || (this._dictionaryOffset += curr.length);
 this._dictionary = data.dictionary as Vector<T['dictionary']>;
 this.clear();
 return chunk;
 return data;
 }
 public finish() {
 this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
 this.indices.finish();
 this.dictionary.finish();
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 return super.finish();
 }
 public clear() {
 this.indices.clear();
 this.dictionary.clear();
 return super.clear();
 }
 public valueToKey(val: any): string | number {
Expand Down

2 changes: 1 addition & 1 deletion
js/src/column.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

 if (typeof field === 'string') {
 const type = chunks[0].data.type;
 field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
 field = new Field(field, type, true);
 } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
 field = field.clone({ nullable: true });
 }
Expand Down

19 changes: 13 additions & 6 deletions
js/src/data.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
 public readonly offset: number;
 public readonly stride: number;
 public readonly childData: Data[];

 /**
 * The dictionary for this Vector, if any. Only used for Dictionary type.
 */
 public dictionary?: Vector;

 public readonly values: Buffers<T>[BufferType.DATA];
 // @ts-ignore
 public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
 return nullCount;
 }

 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
 this.type = type;
 this.dictionary = dictionary;
 this.offset = Math.floor(Math.max(offset || 0, 0));
 this.length = Math.floor(Math.max(length || 0, 0));
 this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
 }

 public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
 return new Data(type, offset, length, nullCount, buffers, childData);
 return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
 }

 public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
 // Convenience methods for creating Data instances for each of the Arrow Vector types
 //
 /** @nocollapse */
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
 if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
 switch (type.typeId) {
 case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
 case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
 case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
 }
 /** @nocollapse */
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
 }
 /** @nocollapse */
 public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down

4 changes: 2 additions & 2 deletions
js/src/interfaces.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
 T extends VectorType ? VectorCtorType<T> :
 T extends VectorType ? VectorCtorType<T> :
 T extends Type ? VectorCtorType<VectorType<T>> :
 T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
 VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
 T extends DataType ? ConstructorType<T> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends Type ? ConstructorType<TypeToDataType<T>> :
 never
 ;
Expand Down

15 changes: 0 additions & 15 deletions
js/src/io/node/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

 constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

 const isDictionary = DataType.isDictionary(builder.type);
 const { queueingStrategy = 'count', autoDestroy = true } = options;
 const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
 this._builder = builder;
 this._desiredSize = highWaterMark;
 this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

 if (isDictionary) {
 let chunks: any[] = [];
 this.push = (chunk: any, _?: string) => {
 if (chunk !== null) {
 chunks.push(chunk);
 return true;
 }
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => super.push(x));
 return super.push(null) && false;
 };
 }
 }
 _read(size: number) {
 this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down

16 changes: 0 additions & 16 deletions
js/src/io/whatwg/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
 'highWaterMark': writableHighWaterMark,
 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
 });

 if (DataType.isDictionary(builderOptions.type)) {
 let chunks: any[] = [];
 this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
 this._bufferedSize = 0;
 if (chunk !== null) {
 chunks.push(chunk);
 } else {
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => controller.enqueue(x));
 controller.close();
 this._controller = null;
 }
 };
 }
 }

 private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down

29 changes: 13 additions & 16 deletions
js/src/ipc/metadata/json.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
 return new Schema(
 schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
 schemaFieldsFromJSON(_schema, dictionaries),
 customMetadataFromJSON(_schema['customMetadata']),
 dictionaries, dictionaryFields
 dictionaries
 );
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

 let id: number;
 let keys: TKeys | null;
 let field: Field | void;
 let dictMeta: any;
 let type: DataType<any>;
 let dictType: Dictionary;
 let dictField: Field<Dictionary>;

 // If no dictionary encoding
 if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
 if (!dictionaries || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
 field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
 else if (!dictionaries.has(id = dictMeta['id'])) {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
 dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.set(id, [field = dictField]);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
 // data type and wrap in a new Dictionary type and field.
 else {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.get(id)!.push(field = dictField);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 return field || null;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

trxcllnt
wants to merge
1
commit into

from

 Closed


ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
#4502
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 2 additions & 2 deletions
js/bin/print-buffer-alignment.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}

16 changes: 2 additions & 14 deletions
js/src/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return function*(source: Iterable<T['TValue'] | TNull>) {
 const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughIterable(options);
 }

 /**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughAsyncIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
 const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughAsyncIterable(options);
 }

 /**
Expand Down

36 changes: 25 additions & 11 deletions
js/src/builder/dictionary.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

 protected _codes = Object.create(null);
 protected _dictionaryOffset: number;
 protected _dictionary?: Vector<T['dictionary']>;
 protected _keysToIndices: { [key: string]: number };
 public readonly indices: IntBuilder<T['indices']>;
 public readonly dictionary: Builder<T['dictionary']>;

 constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
 super({ type });
 super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
 this._nulls = <any> null;
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
 this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
 if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 public get values() { return this.indices.values; }
 public get nullCount() { return this.indices.nullCount; }
 public get nullBitmap() { return this.indices.nullBitmap; }
 public get byteLength() { return this.indices.byteLength; }
 public get reservedLength() { return this.indices.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength; }
 public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
 public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
 public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
 public setValid(index: number, valid: boolean) {
 const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 return valid;
 }
 public setValue(index: number, value: T['TValue']) {
 let keysToCodesMap = this._codes;
 let keysToIndices = this._keysToIndices;
 let key = this.valueToKey(value);
 let idx = keysToCodesMap[key];
 let idx = keysToIndices[key];
 if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
 }
 return this.indices.setValue(index, idx);
 }
 public flush() {
 const chunk = this.indices.flush().clone(this.type);
 const type = this.type;
 const prev = this._dictionary;
 const curr = this.dictionary.toVector();
 const data = this.indices.flush().clone(type);
 data.dictionary = prev ? prev.concat(curr) : curr;
 this.finished || (this._dictionaryOffset += curr.length);
 this._dictionary = data.dictionary as Vector<T['dictionary']>;
 this.clear();
 return chunk;
 return data;
 }
 public finish() {
 this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
 this.indices.finish();
 this.dictionary.finish();
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 return super.finish();
 }
 public clear() {
 this.indices.clear();
 this.dictionary.clear();
 return super.clear();
 }
 public valueToKey(val: any): string | number {
Expand Down

2 changes: 1 addition & 1 deletion
js/src/column.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

 if (typeof field === 'string') {
 const type = chunks[0].data.type;
 field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
 field = new Field(field, type, true);
 } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
 field = field.clone({ nullable: true });
 }
Expand Down

19 changes: 13 additions & 6 deletions
js/src/data.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
 public readonly offset: number;
 public readonly stride: number;
 public readonly childData: Data[];

 /**
 * The dictionary for this Vector, if any. Only used for Dictionary type.
 */
 public dictionary?: Vector;

 public readonly values: Buffers<T>[BufferType.DATA];
 // @ts-ignore
 public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
 return nullCount;
 }

 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
 this.type = type;
 this.dictionary = dictionary;
 this.offset = Math.floor(Math.max(offset || 0, 0));
 this.length = Math.floor(Math.max(length || 0, 0));
 this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
 }

 public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
 return new Data(type, offset, length, nullCount, buffers, childData);
 return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
 }

 public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
 // Convenience methods for creating Data instances for each of the Arrow Vector types
 //
 /** @nocollapse */
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
 if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
 switch (type.typeId) {
 case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
 case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
 case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
 }
 /** @nocollapse */
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
 }
 /** @nocollapse */
 public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down

4 changes: 2 additions & 2 deletions
js/src/interfaces.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
 T extends VectorType ? VectorCtorType<T> :
 T extends VectorType ? VectorCtorType<T> :
 T extends Type ? VectorCtorType<VectorType<T>> :
 T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
 VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
 T extends DataType ? ConstructorType<T> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends Type ? ConstructorType<TypeToDataType<T>> :
 never
 ;
Expand Down

15 changes: 0 additions & 15 deletions
js/src/io/node/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

 constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

 const isDictionary = DataType.isDictionary(builder.type);
 const { queueingStrategy = 'count', autoDestroy = true } = options;
 const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
 this._builder = builder;
 this._desiredSize = highWaterMark;
 this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

 if (isDictionary) {
 let chunks: any[] = [];
 this.push = (chunk: any, _?: string) => {
 if (chunk !== null) {
 chunks.push(chunk);
 return true;
 }
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => super.push(x));
 return super.push(null) && false;
 };
 }
 }
 _read(size: number) {
 this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down

16 changes: 0 additions & 16 deletions
js/src/io/whatwg/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
 'highWaterMark': writableHighWaterMark,
 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
 });

 if (DataType.isDictionary(builderOptions.type)) {
 let chunks: any[] = [];
 this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
 this._bufferedSize = 0;
 if (chunk !== null) {
 chunks.push(chunk);
 } else {
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => controller.enqueue(x));
 controller.close();
 this._controller = null;
 }
 };
 }
 }

 private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down

29 changes: 13 additions & 16 deletions
js/src/ipc/metadata/json.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
 return new Schema(
 schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
 schemaFieldsFromJSON(_schema, dictionaries),
 customMetadataFromJSON(_schema['customMetadata']),
 dictionaries, dictionaryFields
 dictionaries
 );
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

 let id: number;
 let keys: TKeys | null;
 let field: Field | void;
 let dictMeta: any;
 let type: DataType<any>;
 let dictType: Dictionary;
 let dictField: Field<Dictionary>;

 // If no dictionary encoding
 if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
 if (!dictionaries || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
 field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
 else if (!dictionaries.has(id = dictMeta['id'])) {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
 dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.set(id, [field = dictField]);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
 // data type and wrap in a new Dictionary type and field.
 else {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.get(id)!.push(field = dictField);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 return field || null;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

trxcllnt
wants to merge
1
commit into

from

 Closed


ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
#4502
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 2 additions & 2 deletions
js/bin/print-buffer-alignment.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}

16 changes: 2 additions & 14 deletions
js/src/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return function*(source: Iterable<T['TValue'] | TNull>) {
 const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughIterable(options);
 }

 /**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughAsyncIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
 const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughAsyncIterable(options);
 }

 /**
Expand Down

36 changes: 25 additions & 11 deletions
js/src/builder/dictionary.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

 protected _codes = Object.create(null);
 protected _dictionaryOffset: number;
 protected _dictionary?: Vector<T['dictionary']>;
 protected _keysToIndices: { [key: string]: number };
 public readonly indices: IntBuilder<T['indices']>;
 public readonly dictionary: Builder<T['dictionary']>;

 constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
 super({ type });
 super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
 this._nulls = <any> null;
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
 this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
 if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 public get values() { return this.indices.values; }
 public get nullCount() { return this.indices.nullCount; }
 public get nullBitmap() { return this.indices.nullBitmap; }
 public get byteLength() { return this.indices.byteLength; }
 public get reservedLength() { return this.indices.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength; }
 public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
 public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
 public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
 public setValid(index: number, valid: boolean) {
 const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 return valid;
 }
 public setValue(index: number, value: T['TValue']) {
 let keysToCodesMap = this._codes;
 let keysToIndices = this._keysToIndices;
 let key = this.valueToKey(value);
 let idx = keysToCodesMap[key];
 let idx = keysToIndices[key];
 if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
 }
 return this.indices.setValue(index, idx);
 }
 public flush() {
 const chunk = this.indices.flush().clone(this.type);
 const type = this.type;
 const prev = this._dictionary;
 const curr = this.dictionary.toVector();
 const data = this.indices.flush().clone(type);
 data.dictionary = prev ? prev.concat(curr) : curr;
 this.finished || (this._dictionaryOffset += curr.length);
 this._dictionary = data.dictionary as Vector<T['dictionary']>;
 this.clear();
 return chunk;
 return data;
 }
 public finish() {
 this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
 this.indices.finish();
 this.dictionary.finish();
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 return super.finish();
 }
 public clear() {
 this.indices.clear();
 this.dictionary.clear();
 return super.clear();
 }
 public valueToKey(val: any): string | number {
Expand Down

2 changes: 1 addition & 1 deletion
js/src/column.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

 if (typeof field === 'string') {
 const type = chunks[0].data.type;
 field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
 field = new Field(field, type, true);
 } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
 field = field.clone({ nullable: true });
 }
Expand Down

19 changes: 13 additions & 6 deletions
js/src/data.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
 public readonly offset: number;
 public readonly stride: number;
 public readonly childData: Data[];

 /**
 * The dictionary for this Vector, if any. Only used for Dictionary type.
 */
 public dictionary?: Vector;

 public readonly values: Buffers<T>[BufferType.DATA];
 // @ts-ignore
 public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
 return nullCount;
 }

 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
 this.type = type;
 this.dictionary = dictionary;
 this.offset = Math.floor(Math.max(offset || 0, 0));
 this.length = Math.floor(Math.max(length || 0, 0));
 this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
 }

 public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
 return new Data(type, offset, length, nullCount, buffers, childData);
 return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
 }

 public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
 // Convenience methods for creating Data instances for each of the Arrow Vector types
 //
 /** @nocollapse */
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
 if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
 switch (type.typeId) {
 case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
 case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
 case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
 }
 /** @nocollapse */
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
 }
 /** @nocollapse */
 public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down

4 changes: 2 additions & 2 deletions
js/src/interfaces.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
 T extends VectorType ? VectorCtorType<T> :
 T extends VectorType ? VectorCtorType<T> :
 T extends Type ? VectorCtorType<VectorType<T>> :
 T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
 VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
 T extends DataType ? ConstructorType<T> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends Type ? ConstructorType<TypeToDataType<T>> :
 never
 ;
Expand Down

15 changes: 0 additions & 15 deletions
js/src/io/node/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

 constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

 const isDictionary = DataType.isDictionary(builder.type);
 const { queueingStrategy = 'count', autoDestroy = true } = options;
 const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
 this._builder = builder;
 this._desiredSize = highWaterMark;
 this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

 if (isDictionary) {
 let chunks: any[] = [];
 this.push = (chunk: any, _?: string) => {
 if (chunk !== null) {
 chunks.push(chunk);
 return true;
 }
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => super.push(x));
 return super.push(null) && false;
 };
 }
 }
 _read(size: number) {
 this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down

16 changes: 0 additions & 16 deletions
js/src/io/whatwg/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
 'highWaterMark': writableHighWaterMark,
 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
 });

 if (DataType.isDictionary(builderOptions.type)) {
 let chunks: any[] = [];
 this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
 this._bufferedSize = 0;
 if (chunk !== null) {
 chunks.push(chunk);
 } else {
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => controller.enqueue(x));
 controller.close();
 this._controller = null;
 }
 };
 }
 }

 private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down

29 changes: 13 additions & 16 deletions
js/src/ipc/metadata/json.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
 return new Schema(
 schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
 schemaFieldsFromJSON(_schema, dictionaries),
 customMetadataFromJSON(_schema['customMetadata']),
 dictionaries, dictionaryFields
 dictionaries
 );
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

 let id: number;
 let keys: TKeys | null;
 let field: Field | void;
 let dictMeta: any;
 let type: DataType<any>;
 let dictType: Dictionary;
 let dictField: Field<Dictionary>;

 // If no dictionary encoding
 if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
 if (!dictionaries || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
 field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
 else if (!dictionaries.has(id = dictMeta['id'])) {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
 dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.set(id, [field = dictField]);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
 // data type and wrap in a new Dictionary type and field.
 else {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.get(id)!.push(field = dictField);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 return field || null;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder#4502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

trxcllnt
wants to merge
1
commit into

from

 Closed


ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
#4502
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 2 additions & 2 deletions
js/bin/print-buffer-alignment.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });

function loadRecordBatch(schema, header, body) {
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
 return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}

function loadDictionaryBatch(header, body, dictionaryType) {
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
 return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}

16 changes: 2 additions & 14 deletions
js/src/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return function*(source: Iterable<T['TValue'] | TNull>) {
 const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughIterable(options);
 }

 /**
Expand DownExpand Up@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 * @nocollapse
 */
 public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
 const build = throughAsyncIterable(options);
 if (!DataType.isDictionary(options.type)) {
 return build;
 }
 return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
 const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
 };
 return throughAsyncIterable(options);
 }

 /**
Expand Down

36 changes: 25 additions & 11 deletions
js/src/builder/dictionary.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {

 protected _codes = Object.create(null);
 protected _dictionaryOffset: number;
 protected _dictionary?: Vector<T['dictionary']>;
 protected _keysToIndices: { [key: string]: number };
 public readonly indices: IntBuilder<T['indices']>;
 public readonly dictionary: Builder<T['dictionary']>;

 constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
 super({ type });
 super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
 this._nulls = <any> null;
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
 this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
 if (typeof hashFn === 'function') {
Expand All@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 public get values() { return this.indices.values; }
 public get nullCount() { return this.indices.nullCount; }
 public get nullBitmap() { return this.indices.nullBitmap; }
 public get byteLength() { return this.indices.byteLength; }
 public get reservedLength() { return this.indices.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength; }
 public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
 public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
 public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
 public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
 public setValid(index: number, valid: boolean) {
 const indices = this.indices;
Expand All@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
 return valid;
 }
 public setValue(index: number, value: T['TValue']) {
 let keysToCodesMap = this._codes;
 let keysToIndices = this._keysToIndices;
 let key = this.valueToKey(value);
 let idx = keysToCodesMap[key];
 let idx = keysToIndices[key];
 if (idx === undefined) {
keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
 }
 return this.indices.setValue(index, idx);
 }
 public flush() {
 const chunk = this.indices.flush().clone(this.type);
 const type = this.type;
 const prev = this._dictionary;
 const curr = this.dictionary.toVector();
 const data = this.indices.flush().clone(type);
 data.dictionary = prev ? prev.concat(curr) : curr;
 this.finished || (this._dictionaryOffset += curr.length);
 this._dictionary = data.dictionary as Vector<T['dictionary']>;
 this.clear();
 return chunk;
 return data;
 }
 public finish() {
 this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
 this.indices.finish();
 this.dictionary.finish();
 this._dictionaryOffset = 0;
 this._keysToIndices = Object.create(null);
 return super.finish();
 }
 public clear() {
 this.indices.clear();
 this.dictionary.clear();
 return super.clear();
 }
 public valueToKey(val: any): string | number {
Expand Down

2 changes: 1 addition & 1 deletion
js/src/column.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>

 if (typeof field === 'string') {
 const type = chunks[0].data.type;
 field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
 field = new Field(field, type, true);
 } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
 field = field.clone({ nullable: true });
 }
Expand Down

19 changes: 13 additions & 6 deletions
js/src/data.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
 public readonly offset: number;
 public readonly stride: number;
 public readonly childData: Data[];

 /**
 * The dictionary for this Vector, if any. Only used for Dictionary type.
 */
 public dictionary?: Vector;

 public readonly values: Buffers<T>[BufferType.DATA];
 // @ts-ignore
 public readonly typeIds: Buffers<T>[BufferType.TYPE];
Expand DownExpand Up@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
 return nullCount;
 }

 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
 constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
 this.type = type;
 this.dictionary = dictionary;
 this.offset = Math.floor(Math.max(offset || 0, 0));
 this.length = Math.floor(Math.max(length || 0, 0));
 this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
Expand All@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
 }

 public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
 return new Data(type, offset, length, nullCount, buffers, childData);
 return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
 }

 public slice(offset: number, length: number): Data<T> {
Expand DownExpand Up@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
 // Convenience methods for creating Data instances for each of the Arrow Vector types
 //
 /** @nocollapse */
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
 public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
 if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
 switch (type.typeId) {
 case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
 case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
 case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
 case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
Expand DownExpand Up@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
 }
 /** @nocollapse */
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
 public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
 return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
 }
 /** @nocollapse */
 public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
Expand Down

4 changes: 2 additions & 2 deletions
js/src/interfaces.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =

/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
 T extends VectorType ? VectorCtorType<T> :
 T extends VectorType ? VectorCtorType<T> :
 T extends Type ? VectorCtorType<VectorType<T>> :
 T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
 VectorCtorType<vecs.BaseVector>
Expand All@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
 T extends DataType ? ConstructorType<T> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends VectorType ? ConstructorType<T['type']> :
 T extends Type ? ConstructorType<TypeToDataType<T>> :
 never
 ;
Expand Down

15 changes: 0 additions & 15 deletions
js/src/io/node/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {

 constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {

 const isDictionary = DataType.isDictionary(builder.type);
 const { queueingStrategy = 'count', autoDestroy = true } = options;
 const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;

Expand All@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
 this._builder = builder;
 this._desiredSize = highWaterMark;
 this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;

 if (isDictionary) {
 let chunks: any[] = [];
 this.push = (chunk: any, _?: string) => {
 if (chunk !== null) {
 chunks.push(chunk);
 return true;
 }
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => super.push(x));
 return super.push(null) && false;
 };
 }
 }
 _read(size: number) {
 this._maybeFlush(this._builder, this._desiredSize = size);
Expand Down

16 changes: 0 additions & 16 deletions
js/src/io/whatwg/builder.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
 'highWaterMark': writableHighWaterMark,
 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
 });

 if (DataType.isDictionary(builderOptions.type)) {
 let chunks: any[] = [];
 this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
 this._bufferedSize = 0;
 if (chunk !== null) {
 chunks.push(chunk);
 } else {
 const chunks_ = chunks;
 chunks = [];
 chunks_.forEach((x) => controller.enqueue(x));
 controller.close();
 this._controller = null;
 }
 };
 }
 }

 private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
Expand Down

29 changes: 13 additions & 16 deletions
js/src/ipc/metadata/json.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';

/** @ignore */
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
 return new Schema(
 schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
 schemaFieldsFromJSON(_schema, dictionaries),
 customMetadataFromJSON(_schema['customMetadata']),
 dictionaries, dictionaryFields
 dictionaries
 );
}

Expand All@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}

/** @ignore */
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
 return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
 return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}

/** @ignore */
Expand DownExpand Up@@ -93,19 +93,18 @@ function nullCountFromJSON(validity: number[]) {
}

/** @ignore */
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {

 let id: number;
 let keys: TKeys | null;
 let field: Field | void;
 let dictMeta: any;
 let type: DataType<any>;
 let dictType: Dictionary;
 let dictField: Field<Dictionary>;

 // If no dictionary encoding
 if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
 if (!dictionaries || !(dictMeta = _field['dictionary'])) {
 type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
 field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // tslint:disable
Expand All@@ -115,19 +114,17 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
 else if (!dictionaries.has(id = dictMeta['id'])) {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
 dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
 dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.set(id, [field = dictField]);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
 // data type and wrap in a new Dictionary type and field.
 else {
 // a dictionary index defaults to signed 32 bit int if unspecified
 keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
 dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
 dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 dictionaryFields.get(id)!.push(field = dictField);
 field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
 }
 return field || null;
}
Expand Down
Loading