diff options
Diffstat (limited to 'pw_tokenizer/ts')
-rw-r--r-- | pw_tokenizer/ts/detokenizer.ts | 39 | ||||
-rw-r--r-- | pw_tokenizer/ts/detokenizer_test.ts | 22 | ||||
-rw-r--r-- | pw_tokenizer/ts/index.ts | 4 | ||||
-rw-r--r-- | pw_tokenizer/ts/int_testdata.ts | 94 | ||||
-rw-r--r-- | pw_tokenizer/ts/printf_decoder.ts | 78 | ||||
-rw-r--r-- | pw_tokenizer/ts/printf_decoder_test.ts | 50 | ||||
-rw-r--r-- | pw_tokenizer/ts/token_database.ts | 6 |
7 files changed, 172 insertions, 121 deletions
diff --git a/pw_tokenizer/ts/detokenizer.ts b/pw_tokenizer/ts/detokenizer.ts index fe6ea910a..8137d3471 100644 --- a/pw_tokenizer/ts/detokenizer.ts +++ b/pw_tokenizer/ts/detokenizer.ts @@ -13,10 +13,10 @@ // the License. /** Decodes and detokenizes strings from binary or Base64 input. */ -import {Buffer} from 'buffer'; -import {Frame} from 'pigweedjs/pw_hdlc'; -import {TokenDatabase} from './token_database'; -import {PrintfDecoder} from './printf_decoder'; +import { Buffer } from 'buffer'; +import { Frame } from 'pigweedjs/pw_hdlc'; +import { TokenDatabase } from './token_database'; +import { PrintfDecoder } from './printf_decoder'; const MAX_RECURSIONS = 9; const BASE64CHARS = '[A-Za-z0-9+/-_]'; @@ -27,7 +27,7 @@ const PATTERN = new RegExp( `(?:${BASE64CHARS}{4})*` + // The last block of 4 chars may have one or two padding chars (=). `(?:${BASE64CHARS}{3}=|${BASE64CHARS}{2}==)?`, - 'g' + 'g', ); interface TokenAndArgs { @@ -61,7 +61,7 @@ export class Detokenizer { * returned as string as-is. */ detokenizeUint8Array(data: Uint8Array): string { - const {token, args} = this.decodeUint8Array(data); + const { token, args } = this.decodeUint8Array(data); // Parse arguments if this is printf-style text. const format = this.database.get(token); if (format) { @@ -80,7 +80,7 @@ export class Detokenizer { */ detokenizeBase64( tokenizedFrame: Frame, - maxRecursion: number = MAX_RECURSIONS + maxRecursion: number = MAX_RECURSIONS, ): string { const base64String = new TextDecoder().decode(tokenizedFrame.data); return this.detokenizeBase64String(base64String, maxRecursion); @@ -88,16 +88,16 @@ export class Detokenizer { private detokenizeBase64String( base64String: string, - recursions: number + recursions: number, ): string { - return base64String.replace(PATTERN, base64Substring => { - const {token, args} = this.decodeBase64TokenFrame(base64Substring); + return base64String.replace(PATTERN, (base64Substring) => { + const { token, args } = this.decodeBase64TokenFrame(base64Substring); const format = this.database.get(token); // Parse arguments if this is printf-style text. if (format) { const decodedOriginal = new PrintfDecoder().decode( String(format), - args + args, ); // Detokenize nested Base64 tokens and their arguments. if (recursions > 0) { @@ -110,14 +110,13 @@ export class Detokenizer { } private decodeUint8Array(data: Uint8Array): TokenAndArgs { - const token = new DataView( - data.buffer, - data.byteOffset, - 4 - ).getUint32(0, true); + const token = new DataView(data.buffer, data.byteOffset, 4).getUint32( + 0, + true, + ); const args = new Uint8Array(data.buffer.slice(data.byteOffset + 4)); - return {token, args}; + return { token, args }; } private decodeBase64TokenFrame(base64Data: string): TokenAndArgs { @@ -125,15 +124,15 @@ export class Detokenizer { const prefixRemoved = base64Data.slice(1); const noBase64 = Buffer.from(prefixRemoved, 'base64').toString('binary'); // Convert back to bytes and return token and arguments. - const bytes = noBase64.split('').map(ch => ch.charCodeAt(0)); + const bytes = noBase64.split('').map((ch) => ch.charCodeAt(0)); const uIntArray = new Uint8Array(bytes); const token = new DataView( uIntArray.buffer, uIntArray.byteOffset, - 4 + 4, ).getUint32(0, true); const args = new Uint8Array(bytes.slice(4)); - return {token, args}; + return { token, args }; } } diff --git a/pw_tokenizer/ts/detokenizer_test.ts b/pw_tokenizer/ts/detokenizer_test.ts index 1be4de0ae..0340f260e 100644 --- a/pw_tokenizer/ts/detokenizer_test.ts +++ b/pw_tokenizer/ts/detokenizer_test.ts @@ -14,8 +14,8 @@ /* eslint-env browser */ -import {Frame, Encoder, Decoder} from 'pigweedjs/pw_hdlc'; -import {Detokenizer} from './detokenizer'; +import { Frame, Encoder, Decoder } from 'pigweedjs/pw_hdlc'; +import { Detokenizer } from './detokenizer'; const CSV = ` 64636261, ,"regular token" @@ -50,18 +50,18 @@ describe('Detokenizer', () => { it('failure to detokenize returns original string', () => { expect(detokenizer.detokenize(generateFrame('aabbcc'))).toEqual('aabbcc'); expect(detokenizer.detokenizeBase64(generateFrame('$8zP7hg=='))).toEqual( - '$8zP7hg==' + '$8zP7hg==', ); }); it('recursive detokenize all nested base64 tokens', () => { expect( detokenizer.detokenizeBase64( generateFrame( - '$PNNrDQkkN1lZZFJRPT0lJGIxNFlsd2trTjFsWlpGSlJQVDBGUTJGdFpXeFlwSENkUHc9PQ==' - ) - ) + '$PNNrDQkkN1lZZFJRPT0lJGIxNFlsd2trTjFsWlpGSlJQVDBGUTJGdFpXeFlwSENkUHc9PQ==', + ), + ), ).toEqual( - 'Regular Token: Cat and Nested Token: (token: Cat, string: Camel, int: 44, float: 1.2300000190734863)' + 'Regular Token: Cat and Nested Token: (token: Cat, string: Camel, int: 44, float: 1.2300000190734863)', ); }); @@ -69,12 +69,12 @@ describe('Detokenizer', () => { expect( detokenizer.detokenizeBase64( generateFrame( - '$PNNrDQkkN1lZZFJRPT0lJGIxNFlsd2trTjFsWlpGSlJQVDBGUTJGdFpXeFlwSENkUHc9PQ==' + '$PNNrDQkkN1lZZFJRPT0lJGIxNFlsd2trTjFsWlpGSlJQVDBGUTJGdFpXeFlwSENkUHc9PQ==', ), - 1 - ) + 1, + ), ).toEqual( - 'Regular Token: Cat and Nested Token: (token: $7YYdRQ==, string: Camel, int: 44, float: 1.2300000190734863)' + 'Regular Token: Cat and Nested Token: (token: $7YYdRQ==, string: Camel, int: 44, float: 1.2300000190734863)', ); }); }); diff --git a/pw_tokenizer/ts/index.ts b/pw_tokenizer/ts/index.ts index 1989b0592..a90ffa7d6 100644 --- a/pw_tokenizer/ts/index.ts +++ b/pw_tokenizer/ts/index.ts @@ -12,5 +12,5 @@ // License for the specific language governing permissions and limitations under // the License. -export {Detokenizer} from './detokenizer'; -export {PrintfDecoder} from './printf_decoder'; +export { Detokenizer } from './detokenizer'; +export { PrintfDecoder } from './printf_decoder'; diff --git a/pw_tokenizer/ts/int_testdata.ts b/pw_tokenizer/ts/int_testdata.ts index 36b5a21be..1fe01b494 100644 --- a/pw_tokenizer/ts/int_testdata.ts +++ b/pw_tokenizer/ts/int_testdata.ts @@ -13,40 +13,64 @@ // the License. const IntDB = [ - ["%d", "-128", "%u", "4294967168", '\xff\x01'], - ["%d", "-10", "%u", "4294967286", '\x13'], - ["%d", "-9", "%u", "4294967287", '\x11'], - ["%d", "-8", "%u", "4294967288", '\x0f'], - ["%d", "-7", "%u", "4294967289", '\x0d'], - ["%d", "-6", "%u", "4294967290", '\x0b'], - ["%d", "-5", "%u", "4294967291", '\x09'], - ["%d", "-4", "%u", "4294967292", '\x07'], - ["%d", "-3", "%u", "4294967293", '\x05'], - ["%d", "-2", "%u", "4294967294", '\x03'], - ["%d", "-1", "%u", "4294967295", '\x01'], - ["%d", "0", "%u", "0", '\x00'], - ["%d", "1", "%u", "1", '\x02'], - ["%d", "2", "%u", "2", '\x04'], - ["%d", "3", "%u", "3", '\x06'], - ["%d", "4", "%u", "4", '\x08'], - ["%d", "5", "%u", "5", '\x0a'], - ["%d", "6", "%u", "6", '\x0c'], - ["%d", "7", "%u", "7", '\x0e'], - ["%d", "8", "%u", "8", '\x10'], - ["%d", "9", "%u", "9", '\x12'], - ["%d", "10", "%u", "10", '\x14'], - ["%d", "127", "%u", "127", '\xfe\x01'], - ["%d", "-32768", "%u", "4294934528", '\xff\xff\x03'], - ["%d", "652344632", "%u", "652344632", '\xf0\xf4\x8f\xee\x04'], - ["%d", "18567", "%u", "18567", '\x8e\xa2\x02'], - ["%d", "-14", "%u", "4294967282", '\x1b'], - ["%d", "-2147483648", "%u", "2147483648", '\xff\xff\xff\xff\x0f'], - ["%ld", "-14", "%lu", "4294967282", '\x1b'], - ["%d", "2075650855", "%u", "2075650855", '\xce\xac\xbf\xbb\x0f'], - ["%lld", "5922204476835468009", "%llu", "5922204476835468009", '\xd2\xcb\x8c\x90\x86\xe6\xf2\xaf\xa4\x01'], - ["%lld", "-9223372036854775808", "%llu", "9223372036854775808", '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01'], - ["%lld", "3273441488341945355", "%llu", "3273441488341945355", '\x96\xb0\xae\x9a\x96\xec\xcc\xed\x5a'], - ["%lld", "-9223372036854775807", "%llu", "9223372036854775809", '\xfd\xff\xff\xff\xff\xff\xff\xff\xff\x01'], -] + ['%d', '-128', '%u', '4294967168', '\xff\x01'], + ['%d', '-10', '%u', '4294967286', '\x13'], + ['%d', '-9', '%u', '4294967287', '\x11'], + ['%d', '-8', '%u', '4294967288', '\x0f'], + ['%d', '-7', '%u', '4294967289', '\x0d'], + ['%d', '-6', '%u', '4294967290', '\x0b'], + ['%d', '-5', '%u', '4294967291', '\x09'], + ['%d', '-4', '%u', '4294967292', '\x07'], + ['%d', '-3', '%u', '4294967293', '\x05'], + ['%d', '-2', '%u', '4294967294', '\x03'], + ['%d', '-1', '%u', '4294967295', '\x01'], + ['%d', '0', '%u', '0', '\x00'], + ['%d', '1', '%u', '1', '\x02'], + ['%d', '2', '%u', '2', '\x04'], + ['%d', '3', '%u', '3', '\x06'], + ['%d', '4', '%u', '4', '\x08'], + ['%d', '5', '%u', '5', '\x0a'], + ['%d', '6', '%u', '6', '\x0c'], + ['%d', '7', '%u', '7', '\x0e'], + ['%d', '8', '%u', '8', '\x10'], + ['%d', '9', '%u', '9', '\x12'], + ['%d', '10', '%u', '10', '\x14'], + ['%d', '127', '%u', '127', '\xfe\x01'], + ['%d', '-32768', '%u', '4294934528', '\xff\xff\x03'], + ['%d', '652344632', '%u', '652344632', '\xf0\xf4\x8f\xee\x04'], + ['%d', '18567', '%u', '18567', '\x8e\xa2\x02'], + ['%d', '-14', '%u', '4294967282', '\x1b'], + ['%d', '-2147483648', '%u', '2147483648', '\xff\xff\xff\xff\x0f'], + ['%ld', '-14', '%lu', '4294967282', '\x1b'], + ['%d', '2075650855', '%u', '2075650855', '\xce\xac\xbf\xbb\x0f'], + [ + '%lld', + '5922204476835468009', + '%llu', + '5922204476835468009', + '\xd2\xcb\x8c\x90\x86\xe6\xf2\xaf\xa4\x01', + ], + [ + '%lld', + '-9223372036854775808', + '%llu', + '9223372036854775808', + '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01', + ], + [ + '%lld', + '3273441488341945355', + '%llu', + '3273441488341945355', + '\x96\xb0\xae\x9a\x96\xec\xcc\xed\x5a', + ], + [ + '%lld', + '-9223372036854775807', + '%llu', + '9223372036854775809', + '\xfd\xff\xff\xff\xff\xff\xff\xff\xff\x01', + ], +]; export default IntDB; diff --git a/pw_tokenizer/ts/printf_decoder.ts b/pw_tokenizer/ts/printf_decoder.ts index 7b0203031..adf815908 100644 --- a/pw_tokenizer/ts/printf_decoder.ts +++ b/pw_tokenizer/ts/printf_decoder.ts @@ -13,9 +13,10 @@ // the License. /** Decodes arguments and formats them with the provided format string. */ -import Long from "long"; +import Long from 'long'; -const SPECIFIER_REGEX = /%(\.([0-9]+))?(hh|h|ll|l|j|z|t|L)?([%csdioxXufFeEaAgGnp])/g; +const SPECIFIER_REGEX = + /%(\.([0-9]+))?(hh|h|ll|l|j|z|t|L)?([%csdioxXufFeEaAgGnp])/g; // Conversion specifiers by type; n is not supported. const SIGNED_INT = 'di'.split(''); const UNSIGNED_INT = 'oxXup'.split(''); @@ -37,7 +38,7 @@ interface DecodedArg { } // ZigZag decode function from protobuf's wire_format module. -function zigzagDecode(value: Long, unsigned: boolean = false): Long { +function zigzagDecode(value: Long, unsigned = false): Long { // 64 bit math is: // signmask = (zigzag & 1) ? -1 : 0; // twosComplement = (zigzag >> 1) ^ signmask; @@ -45,17 +46,18 @@ function zigzagDecode(value: Long, unsigned: boolean = false): Long { // To work with 32 bit, we can operate on both but "carry" the lowest bit // from the high word by shifting it up 31 bits to be the most significant bit // of the low word. - var bitsLow = value.low, bitsHigh = value.high; - var signFlipMask = -(bitsLow & 1); + let bitsLow = value.low, + bitsHigh = value.high; + const signFlipMask = -(bitsLow & 1); bitsLow = ((bitsLow >>> 1) | (bitsHigh << 31)) ^ signFlipMask; bitsHigh = (bitsHigh >>> 1) ^ signFlipMask; return new Long(bitsLow, bitsHigh, unsigned); -}; +} export class PrintfDecoder { // Reads a unicode string from the encoded data. private decodeString(args: Uint8Array): DecodedArg { - if (args.length === 0) return {size: 0, value: null}; + if (args.length === 0) return { size: 0, value: null }; let sizeAndStatus = args[0]; let status = DecodedStatusFlags.OK; @@ -71,32 +73,37 @@ export class PrintfDecoder { } const decoded = new TextDecoder().decode(data); - return {size: rawData.length, value: decoded}; + return { size: rawData.length, value: decoded }; } private decodeSignedInt(args: Uint8Array): DecodedArg { return this._decodeInt(args); } - private _decodeInt(args: Uint8Array, unsigned: boolean = false): DecodedArg { - if (args.length === 0) return {size: 0, value: null}; + private _decodeInt(args: Uint8Array, unsigned = false): DecodedArg { + if (args.length === 0) return { size: 0, value: null }; let count = 0; let result = new Long(0); let shift = 0; for (count = 0; count < args.length; count++) { const byte = args[count]; - result = result.or((Long.fromInt(byte, unsigned).and(0x7f)).shiftLeft(shift)); + result = result.or( + Long.fromInt(byte, unsigned).and(0x7f).shiftLeft(shift), + ); if (!(byte & 0x80)) { - return {value: zigzagDecode(result, unsigned), size: count + 1}; + return { value: zigzagDecode(result, unsigned), size: count + 1 }; } shift += 7; if (shift >= 64) break; } - return {size: 0, value: null}; + return { size: 0, value: null }; } - private decodeUnsignedInt(args: Uint8Array, lengthSpecifier: string): DecodedArg { + private decodeUnsignedInt( + args: Uint8Array, + lengthSpecifier: string, + ): DecodedArg { const arg = this._decodeInt(args, true); const bits = ['ll', 'j'].indexOf(lengthSpecifier) !== -1 ? 64 : 32; @@ -105,9 +112,8 @@ export class PrintfDecoder { if (arg.value !== null) { let num = arg.value as Long; if (bits === 32) { - num = num.and((Long.fromInt(1).shiftLeft(bits)).add(-1)); - } - else { + num = num.and(Long.fromInt(1).shiftLeft(bits).add(-1)); + } else { num = num.and(-1); } arg.value = num.toString(); @@ -126,17 +132,23 @@ export class PrintfDecoder { } private decodeFloat(args: Uint8Array, precision: string): DecodedArg { - if (args.length < 4) return {size: 0, value: ''}; + if (args.length < 4) return { size: 0, value: '' }; const floatValue = new DataView(args.buffer, args.byteOffset, 4).getFloat32( 0, - true + true, ); - if (precision) return {size: 4, value: floatValue.toFixed(parseInt(precision))} - return {size: 4, value: floatValue}; + if (precision) + return { size: 4, value: floatValue.toFixed(parseInt(precision)) }; + return { size: 4, value: floatValue }; } - private format(specifierType: string, args: Uint8Array, precision: string, lengthSpecifier: string): DecodedArg { - if (specifierType == '%') return {size: 0, value: '%'}; // literal % + private format( + specifierType: string, + args: Uint8Array, + precision: string, + lengthSpecifier: string, + ): DecodedArg { + if (specifierType == '%') return { size: 0, value: '%' }; // literal % if (specifierType === 's') { return this.decodeString(args); } @@ -154,17 +166,29 @@ export class PrintfDecoder { } // Unsupported specifier, return as-is - return {size: 0, value: '%' + specifierType}; + return { size: 0, value: '%' + specifierType }; } decode(formatString: string, args: Uint8Array): string { return formatString.replace( SPECIFIER_REGEX, - (_specifier, _precisionFull, precision, lengthSpecifier, specifierType) => { - const decodedArg = this.format(specifierType, args, precision, lengthSpecifier); + ( + _specifier, + _precisionFull, + precision, + lengthSpecifier, + specifierType, + ) => { + const decodedArg = this.format( + specifierType, + args, + precision, + lengthSpecifier, + ); args = args.slice(decodedArg.size); if (decodedArg === null) return ''; return String(decodedArg.value); - }); + }, + ); } } diff --git a/pw_tokenizer/ts/printf_decoder_test.ts b/pw_tokenizer/ts/printf_decoder_test.ts index db28e488a..938484824 100644 --- a/pw_tokenizer/ts/printf_decoder_test.ts +++ b/pw_tokenizer/ts/printf_decoder_test.ts @@ -13,7 +13,7 @@ // the License. /* eslint-env browser */ -import {PrintfDecoder} from './printf_decoder'; +import { PrintfDecoder } from './printf_decoder'; import IntDB from './int_testdata'; function argFromString(arg: string): Uint8Array { @@ -22,7 +22,7 @@ function argFromString(arg: string): Uint8Array { } function argFromStringBinary(arg: string): Uint8Array { - return new Uint8Array(arg.split('').map(ch => ch.charCodeAt(0))); + return new Uint8Array(arg.split('').map((ch) => ch.charCodeAt(0))); } function argsConcat(...args: Uint8Array[]): Uint8Array { @@ -43,13 +43,13 @@ describe('PrintfDecoder', () => { it('formats string correctly', () => { expect(printfDecoder.decode('Hello %s', argFromString('Computer'))).toEqual( - 'Hello Computer' + 'Hello Computer', ); expect( printfDecoder.decode( 'Hello %s and %s', - argsConcat(argFromString('Mac'), argFromString('PC')) - ) + argsConcat(argFromString('Mac'), argFromString('PC')), + ), ).toEqual('Hello Mac and PC'); }); @@ -57,9 +57,12 @@ describe('PrintfDecoder', () => { expect( printfDecoder.decode( 'Hello %s and %u', - argsConcat(argFromString('Computer'), argFromStringBinary('\xff\xff\x03')) - )).toEqual( - 'Hello Computer and 4294934528'); + argsConcat( + argFromString('Computer'), + argFromStringBinary('\xff\xff\x03'), + ), + ), + ).toEqual('Hello Computer and 4294934528'); }); it('formats integers correctly', () => { @@ -67,15 +70,13 @@ describe('PrintfDecoder', () => { const testcase = IntDB[index]; // Test signed expect( - printfDecoder - .decode(testcase[0], argFromStringBinary(testcase[4]))) - .toEqual(testcase[1]); + printfDecoder.decode(testcase[0], argFromStringBinary(testcase[4])), + ).toEqual(testcase[1]); // Test unsigned expect( - printfDecoder - .decode(testcase[2], argFromStringBinary(testcase[4]))) - .toEqual(testcase[3]); + printfDecoder.decode(testcase[2], argFromStringBinary(testcase[4])), + ).toEqual(testcase[3]); } }); @@ -83,8 +84,8 @@ describe('PrintfDecoder', () => { expect( printfDecoder.decode( 'Hello %s and %s', - argsConcat(argFromString('Mac'), argFromString('PC')) - ) + argsConcat(argFromString('Mac'), argFromString('PC')), + ), ).toEqual('Hello Mac and PC'); }); @@ -92,31 +93,34 @@ describe('PrintfDecoder', () => { const arg = argFromStringBinary('\xff\xff\x03'); expect(printfDecoder.decode('Number %d', arg)).toEqual('Number -32768'); expect( - printfDecoder.decode('Numbers %u and %d', argsConcat(arg, arg)) + printfDecoder.decode('Numbers %u and %d', argsConcat(arg, arg)), ).toEqual('Numbers 4294934528 and -32768'); expect(printfDecoder.decode('Growth is %u%', arg)).toEqual( - 'Growth is 4294934528%' + 'Growth is 4294934528%', ); }); it('formats char correctly', () => { expect( - printfDecoder.decode('Battery: 100%c', argFromStringBinary('\x4a')) + printfDecoder.decode('Battery: 100%c', argFromStringBinary('\x4a')), ).toEqual('Battery: 100%'); expect( - printfDecoder.decode('Price: %c97.99', argFromStringBinary('\x48')) + printfDecoder.decode('Price: %c97.99', argFromStringBinary('\x48')), ).toEqual('Price: $97.99'); }); it('formats floats correctly', () => { expect( - printfDecoder.decode('Value: %f', argFromStringBinary('\xdb\x0f\x49\x40')) + printfDecoder.decode( + 'Value: %f', + argFromStringBinary('\xdb\x0f\x49\x40'), + ), ).toEqual('Value: 3.1415927410125732'); expect( printfDecoder.decode( 'Value: %.5f', - argFromStringBinary('\xdb\x0f\x49\x40') - ) + argFromStringBinary('\xdb\x0f\x49\x40'), + ), ).toEqual('Value: 3.14159'); }); }); diff --git a/pw_tokenizer/ts/token_database.ts b/pw_tokenizer/ts/token_database.ts index bc5bcf56f..ab2b2f8a6 100644 --- a/pw_tokenizer/ts/token_database.ts +++ b/pw_tokenizer/ts/token_database.ts @@ -31,7 +31,7 @@ export class TokenDatabase { private parseTokensToTokensMap(csv: string[]) { for (const [lineNumber, line] of Object.entries( - csv.map(line => line.split(/,/)) + csv.map((line) => line.split(/,/)), )) { if (!line[0] || !line[2]) { continue; @@ -41,8 +41,8 @@ export class TokenDatabase { console.error( new Error( `TokenDatabase number ${line[0]} at line ` + - `${lineNumber} is not a valid hex number` - ) + `${lineNumber} is not a valid hex number`, + ), ); continue; } |