Skip to content

Commit

Permalink
[Issue #41] - Slimmed down FixedTFramedTransport class (#46)
Browse files Browse the repository at this point in the history
* Removed fixedTTransport class

This may not be necessary, since we have upgraded Thrift to a version above 0.9.2.

See https://github.com/apache/thrift/blob/master/CHANGES.md\#092

* Re-adding TFrameTransport patch so  is supported.
  • Loading branch information
dopatraman authored Mar 16, 2022
1 parent ce1f98b commit eb029bd
Showing 1 changed file with 8 additions and 22 deletions.
30 changes: 8 additions & 22 deletions lib/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,11 @@ import fs, { WriteStream } from 'fs'
import * as parquet_thrift from '../gen-nodejs/parquet_types'
import { NewFileMetaData } from './types/types'

/** We need to use a patched version of TFramedTransport where
* readString returns the original buffer instead of a string if the
* buffer can not be safely encoded as utf8 (see http://bit.ly/2GXeZEF)
*/


type Enums = typeof parquet_thrift.Encoding | typeof parquet_thrift.FieldRepetitionType | typeof parquet_thrift.Type | typeof parquet_thrift.CompressionCodec | typeof parquet_thrift.PageType | typeof parquet_thrift.ConvertedType;

type ThriftObject = NewFileMetaData | parquet_thrift.PageHeader | parquet_thrift.BloomFilterHeader | parquet_thrift.OffsetIndex | parquet_thrift.ColumnIndex | NewFileMetaData;

// May not be needed anymore, Issue at https://github.com/LibertyDSNP/parquetjs/issues/41
/**
* We need to patch Thrift's TFramedTransport class bc the TS type definitions
* do not define a `readPos` field, even though the class implementation has
* one.
*/
class fixedTFramedTransport extends thrift.TFramedTransport {
inBuf: Buffer
readPos: number
Expand All @@ -23,19 +17,11 @@ class fixedTFramedTransport extends thrift.TFramedTransport {
this.inBuf = inBuf
this.readPos = 0
}

readString(len = 0): string {
this.ensureAvailable(len);
var buffer = this.inBuf.slice(this.readPos, this.readPos + len);
var str = this.inBuf.toString('utf8', this.readPos, this.readPos + len);
this.readPos += len;
//@ts-ignore
return (Buffer.from(str).equals(buffer)) ? str : buffer;
}
}


/** Patch PageLocation to be three element array that has getters/setters
type Enums = typeof parquet_thrift.Encoding | typeof parquet_thrift.FieldRepetitionType | typeof parquet_thrift.Type | typeof parquet_thrift.CompressionCodec | typeof parquet_thrift.PageType | typeof parquet_thrift.ConvertedType;

type ThriftObject = NewFileMetaData | parquet_thrift.PageHeader | parquet_thrift.BloomFilterHeader | parquet_thrift.OffsetIndex | parquet_thrift.ColumnIndex | NewFileMetaData;/** Patch PageLocation to be three element array that has getters/setters
* for each of the properties (offset, compressed_page_size, first_row_index)
* This saves space considerably as we do not need to store the full variable
* names for every PageLocation
Expand Down

0 comments on commit eb029bd

Please sign in to comment.