Spaces:
Running
Running
File size: 3,464 Bytes
78c921d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import { Duplex, DuplexOptions } from 'stream';
import { AsyncByteQueue } from '../../io/stream.js';
import { RecordBatchReader } from '../../ipc/reader.js';
import { RecordBatch } from '../../recordbatch.js';
import { TypeMap } from '../../type.js';
/** @ignore */
export function recordBatchReaderThroughNodeStream<T extends TypeMap = any>(options?: DuplexOptions & { autoDestroy: boolean }) {
return new RecordBatchReaderDuplex<T>(options);
}
/** @ignore */
type CB = (error?: Error | null | undefined) => void;
/** @ignore */
class RecordBatchReaderDuplex<T extends TypeMap = any> extends Duplex {
private _pulling = false;
private _autoDestroy = true;
private _reader: RecordBatchReader | null;
private _asyncQueue: AsyncByteQueue | null;
constructor(options?: DuplexOptions & { autoDestroy: boolean }) {
super({ allowHalfOpen: false, ...options, readableObjectMode: true, writableObjectMode: false });
this._reader = null;
this._pulling = false;
this._asyncQueue = new AsyncByteQueue();
this._autoDestroy = options && (typeof options.autoDestroy === 'boolean') ? options.autoDestroy : true;
}
_final(cb?: CB) {
const aq = this._asyncQueue;
aq?.close();
cb && cb();
}
_write(x: any, _: string, cb: CB) {
const aq = this._asyncQueue;
aq?.write(x);
cb && cb();
return true;
}
_read(size: number) {
const aq = this._asyncQueue;
if (aq && !this._pulling && (this._pulling = true)) {
(async () => {
if (!this._reader) {
this._reader = await this._open(aq);
}
this._pulling = await this._pull(size, this._reader);
})();
}
}
_destroy(err: Error | null, cb: (error: Error | null) => void) {
const aq = this._asyncQueue;
if (aq) { err ? aq.abort(err) : aq.close(); }
cb(this._asyncQueue = this._reader = null);
}
async _open(source: AsyncByteQueue) {
return await (await RecordBatchReader.from<T>(source)).open({ autoDestroy: this._autoDestroy });
}
async _pull(size: number, reader: RecordBatchReader<T>) {
let r: IteratorResult<RecordBatch<T>> | null = null;
while (this.readable && !(r = await reader.next()).done) {
if (!this.push(r.value) || (size != null && --size <= 0)) { break; }
}
if (!this.readable || (r?.done && (reader.autoDestroy || (await reader.reset().open()).closed))) {
this.push(null);
await reader.cancel();
}
return !this.readable;
}
}
|