diff --git a/CHANGES.txt b/CHANGES.txt index 76b65087519..178937f3bdf 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -61,6 +61,8 @@ Avro 1.8.0 (10 August 2014) AVRO-1672. Java: Add date/time logical types and conversions. (blue) + AVRO-1747. JS: Add Javascript IO implementation. (Matthieu Monsch via blue) + OPTIMIZATIONS IMPROVEMENTS diff --git a/lang/js/LICENSE b/lang/js/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/lang/js/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lang/js/NOTICE b/lang/js/NOTICE new file mode 100644 index 00000000000..7e4a66810c6 --- /dev/null +++ b/lang/js/NOTICE @@ -0,0 +1,5 @@ +Apache Avro +Copyright 2011-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/lang/js/README.md b/lang/js/README.md new file mode 100644 index 00000000000..ee454336b23 --- /dev/null +++ b/lang/js/README.md @@ -0,0 +1,103 @@ + + + +# Avro-js + +Pure JavaScript implementation of the [Avro specification](https://avro.apache.org/docs/current/spec.html). + + +## Features + ++ Fast! Typically twice as fast as JSON with much smaller encodings. ++ Full Avro support, including recursive schemas, sort order, and evolution. ++ Serialization of arbitrary JavaScript objects via logical types. ++ Unopinionated 64-bit integer compatibility. ++ No dependencies, `avro-js` even runs in the browser. + + +## Installation + +```bash +$ npm install avro-js +``` + +`avro-js` is compatible with all versions of [node.js][] since `0.11` and major +browsers via [browserify][]. + + +## Documentation + +See `doc/` folder. + + +## Examples + +Inside a node.js module, or using browserify: + +```javascript +var avro = require('avro-js'); +``` + ++ Encode and decode objects: + + ```javascript + // We can declare a schema inline: + var type = avro.parse({ + name: 'Pet', + type: 'record', + fields: [ + {name: 'kind', type: {name: 'Kind', type: 'enum', symbols: ['CAT', 'DOG']}}, + {name: 'name', type: 'string'} + ] + }); + var pet = {kind: 'CAT', name: 'Albert'}; + var buf = type.toBuffer(pet); // Serialized object. + var obj = type.fromBuffer(buf); // {kind: 'CAT', name: 'Albert'} + ``` + ++ Generate random instances of a schema: + + ```javascript + // We can also parse a JSON-stringified schema: + var type = avro.parse('{"type": "fixed", "name": "Id", "size": 4}'); + var id = type.random(); // E.g. Buffer([48, 152, 2, 123]) + ``` + ++ Check whether an object fits a given schema: + + ```javascript + // Or we can specify a path to a schema file (not in the browser): + var type = avro.parse('./Person.avsc'); + var person = {name: 'Bob', address: {city: 'Cambridge', zip: '02139'}}; + var status = type.isValid(person); // Boolean status. + ``` + ++ Get a [readable stream][readable-stream] of decoded records from an Avro + container file (not in the browser): + + ```javascript + avro.createFileDecoder('./records.avro') + .on('metadata', function (type) { /* `type` is the writer's type. */ }) + .on('data', function (record) { /* Do something with the record. */ }); + ``` + + +[node.js]: https://nodejs.org/en/ +[readable-stream]: https://nodejs.org/api/stream.html#stream_class_stream_readable +[browserify]: http://browserify.org/ diff --git a/lang/js/build.sh b/lang/js/build.sh index 9e7073cb3c4..c551f9dac11 100755 --- a/lang/js/build.sh +++ b/lang/js/build.sh @@ -20,21 +20,21 @@ set -e cd `dirname "$0"` case "$1" in - test) - npm install - grunt test - ;; - - dist) - ;; - - clean) - ;; - - *) - echo "Usage: $0 {test|dist|clean}" - exit 1 - + test) + npm install + npm test + ;; + dist) + npm pack + mkdir -p ../../dist/js + mv avro-js-*.tgz ../../dist/js + ;; + clean) + rm -rf node_modules + ;; + *) + echo "Usage: $0 {test|dist|clean}" >&2 + exit 1 esac exit 0 diff --git a/lang/js/doc/API.md b/lang/js/doc/API.md new file mode 100644 index 00000000000..29d973c52ba --- /dev/null +++ b/lang/js/doc/API.md @@ -0,0 +1,758 @@ + + + ++ [Parsing schemas](#parsing-schemas) ++ [Avro types](#avro-types) ++ [Records](#records) ++ [Files and streams](#files-and-streams) + + +## Parsing schemas + +### `parse(schema, [opts])` + ++ `schema` {Object|String} An Avro schema, represented by one of: + + A string containing a JSON-stringified schema (e.g. `'["null", "int"]'`). + + A path to a file containing a JSON-stringified schema (e.g. + `'./Schema.avsc'`). + + A decoded schema object (e.g. `{type: 'array', items: 'int'}`). ++ `opts` {Object} Parsing options. The following keys are currently supported: + + `logicalTypes` {Object} Optional dictionary of + [`LogicalType`](#class-logicaltypeattrs-opts-types). This can be used to + support serialization and deserialization of arbitrary native objects. + + `namespace` {String} Optional parent namespace. + + `registry` {Object} Optional registry of predefined type names. This can + for example be used to override the types used for primitives. + + `typeHook(attrs, opts)` {Function} Function called before each new type is + instantiated. The relevant decoded schema is available as first argument + and the parsing options as second. This function can optionally return a + type which will then be used in place of the result of parsing `schema`. + See below for more details. + +Parse a schema and return an instance of the corresponding +[`Type`](#class-type). + +Using the `typeHook` option, it is possible to customize the parsing process by +intercepting the creation of any type. As a sample use-case, we show below how +to replace the default `EnumType` (which decodes `enum` values into strings) +with a `LongType` (which will decode the `enum`'s values into integers). This +can be useful when the `enum` already exists as a JavaScript object (e.g. if it +was generated by TypeScript). + +```javascript +var longType = new avro.types.LongType(); +function typeHook(schema) { + if (schema.type === 'enum') { + // For simplicity, we don't do any bound checking here but we could by + // implementing a "bounded long" logical type and returning that instead. + return longType(); + } + // Falling through will cause the default type to be used. +} +``` + +To use it: + +```javascript +// Assume we already have an "enum" with each symbol. +var PETS = {CAT: 0, DOG: 1}; + +// We can provide our hook when parsing a schema. +var type = avro.parse({ + name: 'Pet', + type: 'enum', + symbols: ['CAT', 'DOG'] +}, {typeHook: typeHook}); + +// And encode integer enum values directly. +var buf = type.toBuffer(PETS.CAT); +``` + +Finally, type hooks work well with logical types (for example to dynamically +add `logicalType` attributes to a schema). + + +## Avro types + +All the classes below are available in the `avro.types` namespace: + ++ [`Type`](#class-type) ++ Primitive types: + + `BooleanType` + + `BytesType` + + `DoubleType` + + `FloatType` + + `IntType` + + [`LongType`](#class-longtypeattrs-opts) + + `NullType` + + `StringType` ++ Complex types: + + [`ArrayType`](#class-arraytypeattrs-opts) + + [`EnumType`](#class-enumtypeattrs-opts) + + [`FixedType`](#class-fixedtypeattrs-opts) + + [`MapType`](#class-maptypeattrs-opts) + + [`RecordType`](#class-recordtypeattrs-opts) + + [`UnionType`](#class-uniontypeattrs-opts) ++ [`LogicalType`](#class-logicaltypeattrs-opts-types) + + +### Class `Type` + +"Abstract" base Avro type class; all implementations inherit from it. + +##### `type.decode(buf, [pos,] [resolver])` + ++ `buf` {Buffer} Buffer to read from. ++ `pos` {Number} Offset to start reading from. ++ `resolver` {Resolver} Optional resolver to decode values serialized from + another schema. See [`createResolver`](#typecreateresolverwritertype) for how + to create one. + +Returns `{value: value, offset: offset}` if `buf` contains a valid encoding of +`type` (`value` being the decoded value, and `offset` the new offset in the +buffer). Returns `{value: undefined, offset: -1}` when the buffer is too short. + +##### `type.encode(val, buf, [pos])` + ++ `val` {...} The value to encode. An error will be raised if this isn't a + valid `type` value. ++ `buf` {Buffer} Buffer to write to. ++ `pos` {Number} Offset to start writing at. + +Encode a value into an existing buffer. If enough space was available in `buf`, +returns the new (non-negative) offset, otherwise returns `-N` where `N` is the +(positive) number of bytes by which the buffer was short. + +##### `type.fromBuffer(buf, [resolver,] [noCheck])` + ++ `buf` {Buffer} Bytes containing a serialized value of `type`. ++ `resolver` {Resolver} To decode values serialized from another schema. See + [`createResolver`](#typecreateresolverwritertype) for how to create an + resolver. ++ `noCheck` {Boolean} Do not check that the entire buffer has been read. This + can be useful when using an resolver which only decodes fields at the start of + the buffer, allowing decoding to bail early and yield significant performance + speedups. + +Deserialize a buffer into its corresponding value. + +##### `type.toBuffer(val)` + ++ `val` {...} The value to encode. It must be a valid `type` value. + +Returns a `Buffer` containing the Avro serialization of `val`. + +##### `type.fromString(str)` + ++ `str` {String} String representing a JSON-serialized object. + +Deserialize a JSON-encoded object of `type`. + +##### `type.toString([val])` + ++ `val` {...} The value to serialize. If not specified, this method will return + a human-friendly description of `type`. + +Serialize an object into a JSON-encoded string. + +##### `type.isValid(val, [opts])` + ++ `val` {...} The value to validate. ++ `opts` {Object} Options: + + `errorHook(path, any, type)` {Function} Function called when an invalid + value is encountered. When an invalid value causes its parent values to + also be invalid, the latter do not trigger a callback. `path` will be an + array of strings identifying where the mismatch occurred. See below for a + few examples. + +Check whether `val` is a valid `type` value. + +For complex schemas, it can be difficult to figure out which part(s) of `val` +are invalid. The `errorHook` option provides access to more information about +these mismatches. We illustrate a few use-cases below: + +```javascript +// A sample schema. +var personType = avro.parse({ + type: 'record', + name: 'Person', + fields: [ + {name: 'age', type: 'int'}, + {name: 'names', type: {type: 'array', items: 'string'}} + ] +}); + +// A corresponding invalid record. +var invalidPerson = {age: null, names: ['ann', 3, 'bob']}; +``` + +As a first use-case, we use the `errorHook` to implement a function to gather +all invalid paths a value (if any): + +```javascript +function getInvalidPaths(type, val) { + var paths = []; + type.isValid(val, {errorHook: function (path) { paths.push(path.join()); }}); + return paths; +} + +var paths = getInvalidPaths(personType, invalidPerson); // == ['age', 'names,1'] +``` + +We can also implement an `assertValid` function which throws a helpful error on +the first mismatch encountered (if any): + +```javascript +var util = require('util'); + +function assertValid(type, val) { + return type.isValid(val, {errorHook: hook}); + + function hook(path, any) { + throw new Error(util.format('invalid %s: %j', path.join(), any)); + } +} + +try { + assertValid(personType, invalidPerson); // Will throw. +} catch (err) { + // err.message === 'invalid age: null' +} +``` + +##### `type.clone(val, [opts])` + ++ `val` {...} The object to copy. ++ `opts` {Object} Options: + + `coerceBuffers` {Boolean} Allow coercion of JSON buffer representations + into actual `Buffer` objects. + + `fieldHook(field, any, type)` {Function} Function called when each record + field is populated. The value returned by this function will be used + instead of `any`. `field` is the current `Field` instance and `type` the + parent type. + + `wrapUnions` {Boolean} Avro's JSON representation expects all union values + to be wrapped inside objects. Setting this parameter to `true` will try to + wrap unwrapped union values into their first matching type. + +Deep copy a value of `type`. + +##### `type.compare(val1, val2)` + ++ `val1` {...} Value of `type`. ++ `val2` {...} Value of `type`. + +Returns `0` if both values are equal according to their [sort +order][sort-order], `-1` if the first is smaller than the second , and `1` +otherwise. Comparing invalid values is undefined behavior. + +##### `type.compareBuffers(buf1, buf2)` + ++ `buf1` {Buffer} `type` value bytes. ++ `buf2` {Buffer} `type` value bytes. + +Similar to [`compare`](#typecompareval1-val2), but doesn't require decoding +values. + +##### `type.createResolver(writerType)` + ++ `writerType` {Type} Writer type. + +Create a resolver that can be be passed to the `type`'s +[`decode`](#typedecodebuf-pos-resolver) and +[`fromBuffer`](#typefrombufferbuf-resolver-nocheck) methods. This will enable +decoding values which had been serialized using `writerType`, according to the +Avro [resolution rules][schema-resolution]. If the schemas are incompatible, +this method will throw an error. + +For example, assume we have the following two versions of a type: + +```javascript +// A schema's first version. +var v1 = avro.parse({ + name: 'Person', + type: 'record', + fields: [ + {name: 'name', type: 'string'}, + {name: 'age', type: 'int'} + ] +}); + +// The updated version. +var v2 = avro.parse({ + type: 'record', + name: 'Person', + fields: [ + { + name: 'name', type: [ + 'string', + { + name: 'Name', + type: 'record', + fields: [ + {name: 'first', type: 'string'}, + {name: 'last', type: 'string'} + ] + } + ] + }, + {name: 'phone', type: ['null', 'string'], default: null} + ] +}); +``` + +The two types are compatible since the `name` field is present in both (the +`string` can be promoted to the new `union`) and the new `phone` field has a +default value. + +```javascript +// We can therefore create a resolver. +var resolver = v2.createResolver(v1); + +// And pass it whenever we want to decode from the old type to the new. +var buf = v1.toBuffer({name: 'Ann', age: 25}); +var obj = v2.fromBuffer(buf, resolver); // === {name: {string: 'Ann'}, phone: null} +``` + +See the [advanced usage page](Advanced-usage) for more details on how schema +evolution can be used to significantly speed up decoding. + +##### `type.random()` + +Returns a random value of `type`. + +##### `type.getName()` + +Returns `type`'s fully qualified name if it exists, `undefined` otherwise. + +##### `type.getSchema([noDeref])` + ++ `noDeref` {Boolean} Do not dereference any type names. + +Returns `type`'s [canonical schema][canonical-schema] (as a string). This can +be used to compare schemas for equality. + +##### `type.getFingerprint([algorithm])` + ++ `algorithm` {String} Algorithm used to generate the schema's [fingerprint][]. + Defaults to `md5`. *Not supported in the browser.* + +##### `Type.__reset(size)` + ++ `size` {Number} New buffer size in bytes. + +This method resizes the internal buffer used to encode all types. You should +only ever need to call this if you are encoding very large values and need to +reclaim memory. + + +#### Class `LongType(attrs, [opts])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. + +##### `LongType.using(methods, [noUnpack])` + ++ `methods` {Object} Method implementations dictionary keyed by method name, + see below for details on each of the functions to implement. ++ `noUnpack` {Boolean} Do not automatically unpack bytes before passing them to + the above `methods`' `fromBuffer` function and pack bytes returned by its + `toBuffer` function. + +This function provides a way to support arbitrary long representations. Doing +so requires implementing the following methods (a few examples are available +[here][custom-long]): + ++ `fromBuffer(buf)` + + + `buf` {Buffer} Encoded long. If `noUnpack` is off (the default), `buf` will + be an 8-byte buffer containing the long's unpacked representation. + Otherwise, `buf` will contain a variable length buffer with the long's + packed representation. + + This method should return the corresponding decoded long. + ++ `toBuffer(val)` + + + `val` {...} Decoded long. + + If `noUnpack` is off (the default), this method should return an 8-byte + buffer with the `long`'s unpacked representation. Otherwise, `toBuffer` + should return an already packed buffer (of variable length). + ++ `fromJSON(any)` + + + `any` {Number|...} Parsed value. To ensure that the `fromString` method + works correctly on data JSON-serialized according to the Avro spec, this + method should at least support numbers as input. + + This method should return the corresponding decoded long. + + It might also be useful to support other kinds of input (typically the output + of the long implementation's `toJSON` method) to enable serializing large + numbers without loss of precision (at the cost of violating the Avro spec). + ++ `toJSON(val)` + + + `val` {...} Decoded long. + + This method should return the `long`'s JSON representation. + ++ `isValid(val, [opts])` + + See [`Type.isValid`](#typeisvalidval-opts). + ++ `compare(val1, val2)` + + See [`Type.compare`](#typecompareval1-val2). + + +#### Class `ArrayType(attrs, [opts])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. + +##### `type.getItemsType()` + +The type of the array's items. + + +#### Class `EnumType(attrs, [opts])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. + +##### `type.getAliases()` + +Optional type aliases. These are used when adapting a schema from another type. + +##### `type.getSymbols()` + +Returns a copy of the type's symbols (an array of strings representing the +`enum`'s valid values). + + +#### Class `FixedType(attrs, [opts])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. + +##### `type.getAliases()` + +Optional type aliases. These are used when adapting a schema from another type. + +##### `type.getSize()` + +The size in bytes of instances of this type. + + +#### Class `MapType(attrs, [opts])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. + +##### `type.getValuesType()` + +The type of the map's values (keys are always strings). + + +#### Class `RecordType(attrs, [opts])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. + +##### `type.getAliases()` + +Optional type aliases. These are used when adapting a schema from another type. + +##### `type.getFields()` + +Returns a copy of the array of fields contained in this record. Each field is +an object with the following methods: + ++ `getAliases()` ++ `getDefault()` ++ `getName()` ++ `getOrder()` ++ `getType()` + +##### `type.getRecordConstructor()` + +The [`Record`](#class-record) constructor for instances of this type. + + +#### Class `UnionType(attrs, [opts])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. + +##### `type.getTypes()` + +The possible types that this union can take. + + +#### Class `LogicalType(attrs, [opts,] [Types])` + ++ `attrs` {Object} Decoded type attributes. ++ `opts` {Object} Parsing options. ++ `Types` {Array} Optional of type classes. If specified, only these will be + accepted as underlying type. + +"Abstract class" used to implement custom native types. + +##### `type.getUnderlyingType()` + +Get the underlying Avro type. This can be useful when a logical type can +support different underlying types. + +To implement a custom logical type, the steps are: + ++ Call `LogicalType`'s constructor inside your own subclass' to make sure the + underlying type is property set up. Throwing an error anywhere inside your + constructor will prevent the logical type from being used (the underlying + type will be used instead). ++ Extend `LogicalType` in your own subclass (typically using `util.inherits`). ++ Override the methods below (prefixed with an underscore because they are + internal to the class that defines them and should only be called by the + internal `LogicalType` methods). + +See [here][logical-types] for a couple sample implementations. + +##### `type._fromValue(val)` + ++ `val` {...} A value deserialized by the underlying type. + +This function should return the final, wrapped, value. + +##### `type._toValue(any)` + ++ `any` {...} A wrapped value. + +This function should return a value which can be serialized by the underlying +type. + +##### `type._resolve(type)` + ++ `type` {Type} The writer's type. + +This function should return: + ++ `undefined` if the writer's values cannot be converted. ++ Otherwise, a function which converts a value deserialized by the writer's + type into a wrapped value for the current type. + + +## Records + +Each [`RecordType`](#class-recordtypeattrs-opts) generates a corresponding +`Record` constructor when its schema is parsed. It is available using the +`RecordType`'s `getRecordConstructor` methods. This helps make decoding and +encoding records more efficient. + +All prototype methods below are prefixed with `$` to avoid clashing with an +existing record field (`$` is a valid identifier in JavaScript, but not in +Avro). + +#### Class `Record(...)` + +Calling the constructor directly can sometimes be a convenient shortcut to +instantiate new records of a given type. In particular, it will correctly +initialize all the missing record's fields with their default values. + +##### `record.$clone([opts])` + ++ `opts` {Object} See [`type.clone`](#typecloneval-opts). + +Deep copy the record. + +##### `record.$compare(val)` + ++ `val` {Record} See [`type.compare`](#typecompareval1-val2). + +Compare the record to another. + +##### `record.$getType()` + +Get the record's `type`. + +##### `record.$isValid([opts])` + ++ `opts` {Object} See [`type.isValid`](#typeisvalidval-opts). + +Check whether the record is valid. + +##### `record.$toBuffer()` + +Return binary encoding of record. + +##### `record.$toString()` + +Return JSON-stringified record. + +##### `Record.getType()` + +Convenience class method to get the record's type. + + +## Files and streams + +*Not available in the browser.* + +The following convenience functions are available for common operations on +container files: + +#### `createFileDecoder(path, [opts])` + ++ `path` {String} Path to Avro container file. ++ `opts` {Object} Decoding options, passed to + [`BlockDecoder`](Api#class-blockdecoderopts). + +Returns a readable stream of decoded objects from an Avro container file. + +#### `createFileEncoder(path, schema, [opts])` + ++ `path` {String} Destination path. ++ `schem` {Object|String|Type} Type used to serialize. ++ `opts` {Object} Encoding options, passed to + [`BlockEncoder`](Api#class-blockencoderschem-opts). + +Returns a writable stream of objects. These will end up serialized into an Avro +container file. + +#### `extractFileHeader(path, [opts])` + ++ `path` {String} Path to Avro container file. ++ `opts` {Object} Options: + + `decode` {Boolean} Decode schema and codec metadata (otherwise they will be + returned as bytes). Defaults to `true`. + +Extract header from an Avro container file synchronously. If no header is +present (i.e. the path doesn't point to a valid Avro container file), `null` is +returned. + + +For more specific use-cases, the following stream classes are available in the +`avro.streams` namespace: + ++ [`BlockDecoder`](#blockdecoderopts) ++ [`RawDecoder`](#rawdecoderschem-opts) ++ [`BlockEncoder`](#blockencoderschem-opts) ++ [`RawEncoder`](#rawencoderschem-opts) + + +#### Class `BlockDecoder([opts])` + ++ `opts` {Object} Decoding options. Available keys: + + `codecs` {Object} Dictionary of decompression functions, keyed by codec + name. A decompression function has the signature `fn(compressedData, cb)` where + `compressedData` is a buffer of compressed data, and must call `cb(err, + uncompressedData)` on completion. The default contains handlers for the + `'null'` and `'deflate'` codecs. + + `decode` {Boolean} Whether to decode records before returning them. + Defaults to `true`. + + `parseOpts` {Object} Options passed when parsing the writer's schema. + +A duplex stream which decodes bytes coming from on Avro object container file. + +##### Event `'metadata'` + ++ `type` {Type} The type used to write the file. ++ `codec` {String} The codec's name. ++ `header` {Object} The file's header, containing in particular the raw schema + and codec. + +##### Event `'data'` + ++ `data` {...} Decoded element or raw bytes. + +##### `BlockDecoder.getDefaultCodecs()` + +Get built-in decompression functions (currently `null` and `deflate`). + + +#### Class `RawDecoder(schema, [opts])` + ++ `schema` {Object|String|Type} Writer schema. Required since the input doesn't + contain a header. Argument parsing logic is the same as for + [`parse`](#parseschema-opts). ++ `opts` {Object} Decoding options. Available keys: + + `decode` {Boolean} Whether to decode records before returning them. + Defaults to `true`. + +A duplex stream which can be used to decode a stream of serialized Avro objects +with no headers or blocks. + +##### Event `'data'` + ++ `data` {...} Decoded element or raw bytes. + + +#### Class `BlockEncoder(schema, [opts])` + ++ `schema` {Object|String|Type} Schema used for encoding. Argument parsing + logic is the same as for [`parse`](#parseschema-opts). ++ `opts` {Object} Encoding options. Available keys: + + `blockSize` {Number} Maximum uncompressed size of each block data. A new + block will be started when this number is exceeded. If it is too small to + fit a single element, it will be increased appropriately. Defaults to 64kB. + + `codec` {String} Name of codec to use for encoding. See `codecs` option + below to support arbitrary compression functions. + + `codecs` {Object} Dictionary of compression functions, keyed by codec + name. A compression function has the signature `fn(uncompressedData, cb)` where + `uncompressedData` is a buffer of uncompressed data, and must call `cb(err, + compressedData)` on completion. The default contains handlers for the + `'null'` and `'deflate'` codecs. + + `omitHeader` {Boolean} Don't emit the header. This can be useful when + appending to an existing container file. Defaults to `false`. + + `syncMarker` {Buffer} 16 byte buffer to use as synchronization marker + inside the file. If unspecified, a random value will be generated. + +A duplex stream to create Avro container object files. + +##### Event `'data'` + ++ `data` {Buffer} Serialized bytes. + +##### `BlockEncoder.getDefaultCodecs()` + +Get built-in compression functions (currently `null` and `deflate`). + + +#### Class `RawEncoder(schema, [opts])` + ++ `schema` {Object|String|Type} Schema used for encoding. Argument parsing + logic is the same as for [`parse`](#parseschema-opts). ++ `opts` {Object} Encoding options. Available keys: + + `batchSize` {Number} To increase performance, records are serialized in + batches. Use this option to control how often batches are emitted. If it is + too small to fit a single record, it will be increased automatically. + Defaults to 64kB. + +The encoding equivalent of `RawDecoder`. + +##### Event `'data'` + ++ `data` {Buffer} Serialized bytes. + + +[canonical-schema]: https://avro.apache.org/docs/current/spec.html#Parsing+Canonical+Form+for+Schemas +[schema-resolution]: https://avro.apache.org/docs/current/spec.html#Schema+Resolution +[sort-order]: https://avro.apache.org/docs/current/spec.html#order +[fingerprint]: https://avro.apache.org/docs/current/spec.html#Schema+Fingerprints +[custom-long]: Advanced-usage#custom-long-types +[logical-types]: Advanced-usage#logical-types diff --git a/lang/js/doc/Advanced-usage.md b/lang/js/doc/Advanced-usage.md new file mode 100644 index 00000000000..23cab5fab77 --- /dev/null +++ b/lang/js/doc/Advanced-usage.md @@ -0,0 +1,359 @@ + + + ++ [Schema evolution](#schema-evolution) ++ [Logical types](#logical-types) ++ [Custom long types](#custom-long-types) + + +## Schema evolution + +Schema evolution allows a type to deserialize binary data written by another +[compatible][schema-resolution] type. This is done via +[`createResolver`][create-resolver-api], and is particularly useful when we are +only interested in a subset of the fields inside a record. By selectively +decoding fields, we can significantly increase throughput. + +As a motivating example, consider the following event: + +```javascript +var heavyType = avro.parse({ + name: 'Event', + type: 'record', + fields: [ + {name: 'time', type: 'long'}, + {name: 'userId', type: 'int'}, + {name: 'actions', type: {type: 'array', items: 'string'}}, + ] +}); +``` + +Let's assume that we would like to compute statistics on users' actions but +only for a few user IDs. One approach would be to decode the full record each +time, but this is wasteful if very few users match our filter. We can do better +by using the following reader's schema, and creating the corresponding +resolver: + +```javascript +var lightType = avro.parse({ + name: 'LightEvent', + aliases: ['Event'], + type: 'record', + fields: [ + {name: 'userId', type: 'int'}, + ] +}); + +var resolver = lightType.createResolver(heavyType); +``` + +We decode only the `userId` field, and then, if the ID matches, process the +full record. The function below implements this logic, returning a fully +decoded record if the ID matches, and `undefined` otherwise. + +```javascript +function fastDecode(buf) { + var lightRecord = lightType.fromBuffer(buf, resolver, true); + if (lightRecord.userId % 100 === 48) { // Arbitrary check. + return heavyType.fromBuffer(buf); + } +} +``` + +In the above example, using randomly generated records, if the filter matches +roughly 1% of the time, we are able to get a **400%** throughput increase +compared to decoding the full record each time! The heavier the schema (and the +closer to the beginning of the record the used fields are), the higher this +increase will be. + +## Logical types + +The built-in types provided by Avro are sufficient for many use-cases, but it +can often be much more convenient to work with native JavaScript objects. As a +quick motivating example, let's imagine we have the following schema: + +```javascript +var schema = { + name: 'Transaction', + type: 'record', + fields: [ + {name: 'amount', type: 'int'}, + {name: 'time', type: {type: 'long', logicalType: 'timestamp-millis'}} + ] +}; +``` + +The `time` field encodes a timestamp as a `long`, but it would be better if we +could deserialize it directly into a native `Date` object. This is possible +using Avro's *logical types*, with the following two steps: + ++ Adding a `logicalType` attribute to the type's definition (e.g. + `'timestamp-millis'` above). ++ Implementing a corresponding [`LogicalType`][logical-type-api] and adding it + to [`parse`][parse-api]'s `logicalTypes`. + +Below is a sample implementation for a suitable `DateType` which will +transparently deserialize/serialize native `Date` objects: + +```javascript +var util = require('util'); + +function DateType(attrs, opts) { + LogicalType.call(this, attrs, opts, [LongType]); // Require underlying `long`. +} +util.inherits(DateType, LogicalType); + +DateType.prototype._fromValue = function (val) { return new Date(val); }; +DateType.prototype._toValue = function (date) { return +date; }; +``` + +Usage is straightforward: + +```javascript +var type = avro.parse(transactionSchema, {logicalTypes: {date: DateType}}); + +// We create a new transaction. +var transaction = { + amount: 32, + time: new Date('Thu Nov 05 2015 11:38:05 GMT-0800 (PST)') +}; + +// Our type is able to directly serialize it, including the date. +var buf = type.toBuffer(transaction); + +// And we can get the date back just as easily. +var date = type.fromBuffer(buf).time; // `Date` object. +``` + +Logical types can also be used with schema evolution. This is done by +implementing an additional `_resolve` method. It should return a function which +converts values of the writer's type into the logical type's values. For +example, we can allow our `DateType` to read dates which were serialized as +strings: + +```javascript +DateType.prototype._resolve = function (type) { + if ( + type instanceof StringType || // Support parsing strings. + type instanceof LongType || + type instanceof DateType + ) { + return this._fromValue; + } +}; +``` + +And use it as follows: + +```javascript +var stringType = avro.parse('string'); +var str = 'Thu Nov 05 2015 11:38:05 GMT-0800 (PST)'; +var buf = stringType.toBuffer(str); +var resolver = dateType.createResolver(stringType); +var date = dateType.fromBuffer(buf, resolver); // Date corresponding to `str`. +``` + +Finally, as a more fully featured example, we provide a sample implementation +of the [decimal logical type][decimal-type] described in the spec: + +```javascript +/** + * Sample decimal logical type implementation. + * + * It wraps its values in a very simple custom `Decimal` class. + * + */ +function DecimalType(attrs, opts) { + LogicalType.call(this, attrs, opts, [BytesType, FixedType]); + + // Validate attributes. + var precision = attrs.precision; + if (precision !== (precision | 0) || precision <= 0) { + throw new Error('invalid precision'); + } + var scale = attrs.scale; + if (scale !== (scale | 0) || scale < 0 || scale > precision) { + throw new Error('invalid scale'); + } + var type = this.getUnderlyingType(); + if (type instanceof FixedType) { + var size = type.getSize(); + var maxPrecision = Math.log(Math.pow(2, 8 * size - 1) - 1) / Math.log(10); + if (precision > (maxPrecision | 0)) { + throw new Error('fixed size too small to hold required precision'); + } + } + + // A basic decimal class for this precision and scale. + function Decimal(unscaled) { this.unscaled = unscaled; } + Decimal.prototype.precision = precision; + Decimal.prototype.scale = scale; + Decimal.prototype.toNumber = function () { + return this.unscaled * Math.pow(10, -scale); + }; + + this.Decimal = Decimal; +} +util.inherits(DecimalType, LogicalType); + +DecimalType.prototype._fromValue = function (buf) { + return new this.Decimal(buf.readIntBE(0, buf.length)); +}; + +DecimalType.prototype._toValue = function (dec) { + if (!(dec instanceof this.Decimal)) { + throw new Error('invalid decimal'); + } + + var type = this.getUnderlyingType(); + var buf; + if (type instanceof FixedType) { + buf = new Buffer(type.getSize()); + } else { + var size = Math.log(dec > 0 ? dec : - 2 * dec) / (Math.log(2) * 8) | 0; + buf = new Buffer(size + 1); + } + buf.writeIntBE(dec.unscaled, 0, buf.length); + return buf; +}; + +DecimalType.prototype._resolve = function (type) { + if ( + type instanceof DecimalType && + type.Decimal.prototype.precision === this.Decimal.prototype.precision && + type.Decimal.prototype.scale === this.Decimal.prototype.scale + ) { + return function (dec) { return dec; }; + } +}; +``` + + +## Custom long types + +JavaScript represents all numbers as doubles internally, which means that it is +possible to lose precision when using very large numbers (absolute value +greater than `9e+15` or so). For example: + +```javascript +Number.parseInt('9007199254740995') === 9007199254740996 // true +``` + +In most cases, these bounds are so large that this is not a problem (timestamps +fit nicely inside the supported precision). However it might happen that the +full range must be supported. (To avoid silently corrupting data, the default +[`LongType`](Api#class-longtypeschema-opts) will throw an error when +encountering a number outside the supported precision range.) + +There are multiple JavaScript libraries to represent 64-bit integers, with +different characteristics (e.g. some are faster but do not run in the browser). +Rather than tie us to any particular one, `avro` lets us choose the most +adequate with [`LongType.using`](Api#longtypeusingmethods-nounpack). Below +are a few sample implementations for popular libraries (refer to the API +documentation for details on each option; a helper script is also available to +validate our implementation inside `etc/scripts/`): + ++ [`node-int64`](https://www.npmjs.com/package/node-int64): + + ```javascript + var Long = require('node-int64'); + + var longType = avro.types.LongType.using({ + fromBuffer: function (buf) { return new Long(buf); }, + toBuffer: function (n) { return n.toBuffer(); }, + fromJSON: function (obj) { return new Long(obj); }, + toJSON: function (n) { return +n; }, + isValid: function (n) { return n instanceof Long; }, + compare: function (n1, n2) { return n1.compare(n2); } + }); + ``` + ++ [`int64-native`](https://www.npmjs.com/package/int64-native): + + ```javascript + var Long = require('int64-native'); + + var longType = avro.types.LongType.using({ + fromBuffer: function (buf) { return new Long('0x' + buf.toString('hex')); }, + toBuffer: function (n) { return new Buffer(n.toString().slice(2), 'hex'); }, + fromJSON: function (obj) { return new Long(obj); }, + toJSON: function (n) { return +n; }, + isValid: function (n) { return n instanceof Long; }, + compare: function (n1, n2) { return n1.compare(n2); } + }); + ``` + ++ [`long`](https://www.npmjs.com/package/long): + + ```javascript + var Long = require('long'); + + var longType = avro.types.LongType.using({ + fromBuffer: function (buf) { + return new Long(buf.readInt32LE(), buf.readInt32LE(4)); + }, + toBuffer: function (n) { + var buf = new Buffer(8); + buf.writeInt32LE(n.getLowBits()); + buf.writeInt32LE(n.getHighBits(), 4); + return buf; + }, + fromJSON: Long.fromValue, + toJSON: function (n) { return +n; }, + isValid: Long.isLong, + compare: Long.compare + }); + ``` + +Any such implementation can then be used in place of the default `LongType` to +provide full 64-bit support when decoding and encoding binary data. To do so, +we override the default type used for `long`s by adding our implementation to +the `registry` when parsing a schema: + +```javascript +// Our schema here is very simple, but this would work for arbitrarily complex +// ones (applying to all longs inside of it). +var type = avro.parse('long', {registry: {'long': longType}}); + +// Avro serialization of Number.MAX_SAFE_INTEGER + 4 (which is incorrectly +// rounded when represented as a double): +var buf = new Buffer([0x86, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x20]); + +// Assuming we are using the `node-int64` implementation. +var obj = new Long(buf); +var encoded = type.toBuffer(obj); // == buf +var decoded = type.fromBuffer(buf); // == obj (No precision loss.) +``` + +Because the built-in JSON parser is itself limited by JavaScript's internal +number representation, using the `toString` and `fromString` methods is +generally still unsafe (see `LongType.using`'s documentation for a possible +workaround). + +Finally, to make integration easier, `toBuffer` and `fromBuffer` deal with +already unpacked buffers by default. To leverage an external optimized packing +and unpacking routine (for example when using a native C++ addon), we can +disable this behavior by setting `LongType.using`'s `noUnpack` argument to +`true`. + +[parse-api]: API#parseschema-opts +[create-resolver-api]: API#typecreateresolverwritertype +[logical-type-api]: API#class-logicaltypeattrs-opts-types +[decimal-type]: https://avro.apache.org/docs/current/spec.html#Decimal +[schema-resolution]: https://avro.apache.org/docs/current/spec.html#Schema+Resolution diff --git a/lang/js/doc/Home.md b/lang/js/doc/Home.md new file mode 100644 index 00000000000..e152f41d127 --- /dev/null +++ b/lang/js/doc/Home.md @@ -0,0 +1,191 @@ + + + +This page is meant to provide a brief overview of `avro`'s API: + ++ [What is a `Type`?](#what-is-a-type) ++ [How do I get a `Type`?](#how-do-i-get-a-type) ++ [What about Avro files?](#what-about-avro-files) ++ [Next steps](#next-steps) + + +## What is a `Type`? + +Each Avro type maps to a corresponding JavaScript [`Type`](API#class-type): + ++ `int` maps to `IntType`. ++ `array`s map to `ArrayType`s. ++ `record`s map to `RecordType`s. ++ etc. + +An instance of a `Type` knows how to [`decode`](Api#typedecodebuf-pos-resolver) +and [`encode`](Api#typeencodeval-buf-pos) and its corresponding objects. For +example the `StringType` knows how to handle JavaScript strings: + +```javascript +var stringType = new avro.types.StringType(); +var buf = stringType.toBuffer('Hi'); // Buffer containing 'Hi''s Avro encoding. +var str = stringType.fromBuffer(buf); // === 'Hi' +``` + +The [`toBuffer`](API#typetobufferval) and +[`fromBuffer`](API#typefrombufferval-resolver-nocheck) methods above are +convenience functions which encode and decode a single object into/from a +standalone buffer. + +Each `type` also provides other methods which can be useful. Here are a few +(refer to the [API documentation](API#avro-types) for the full list): + ++ JSON-encoding: + + ```javascript + var jsonString = type.toString('Hi'); // === '"Hi"' + var str = type.fromString(jsonString); // === 'Hi' + ``` + ++ Validity checks: + + ```javascript + var b1 = stringType.isValid('hello'); // === true ('hello' is a valid string.) + var b2 = stringType.isValid(-2); // === false (-2 is not.) + ``` + ++ Random object generation: + + ```javascript + var s = stringType.random(); // A random string. + ``` + + +## How do I get a `Type`? + +It is possible to instantiate types directly by calling their constructors +(available in the `avro.types` namespace; this is what we used earlier), but in +the vast majority of use-cases they will be automatically generated by parsing +an existing schema. + +`avro` exposes a [`parse`](Api#parseschema-opts) method to do the +heavy lifting: + +```javascript +// Equivalent to what we did earlier. +var stringType = avro.parse({type: 'string'}); + +// A slightly more complex type. +var mapType = avro.parse({type: 'map', values: 'long'}); + +// The sky is the limit! +var personType = avro.parse({ + name: 'Person', + type: 'record', + fields: [ + {name: 'name', type: 'string'}, + {name: 'phone', type: ['null', 'string'], default: null}, + {name: 'address', type: { + name: 'Address', + type: 'record', + fields: [ + {name: 'city', type: 'string'}, + {name: 'zip', type: 'int'} + ] + }} + ] +}); +``` + +Of course, all the `type` methods are available. For example: + +```javascript +personType.isValid({ + name: 'Ann', + phone: null, + address: {city: 'Cambridge', zip: 02139} +}); // === true + +personType.isValid({ + name: 'Bob', + phone: {string: '617-000-1234'}, + address: {city: 'Boston'} +}); // === false (Missing the zip code.) +``` + +Since schemas are often stored in separate files, passing a path to `parse` +will attempt to load a JSON-serialized schema from there: + +```javascript +var couponType = avro.parse('./Coupon.avsc'); +``` + +For advanced use-cases, `parse` also has a few options which are detailed the +API documentation. + + +## What about Avro files? + +Avro files (meaning [Avro object container files][object-container]) hold +serialized Avro records along with their schema. Reading them is as simple as +calling [`createFileDecoder`](Api#createfiledecoderpath-opts): + +```javascript +var personStream = avro.createFileDecoder('./persons.avro'); +``` + +`personStream` is a [readable stream][rstream] of decoded records, which we can +for example use as follows: + +```javascript +personStream.on('data', function (person) { + if (person.address.city === 'San Francisco') { + doSomethingWith(person); + } +}); +``` + +In case we need the records' `type` or the file's codec, they are available by +listening to the `'metadata'` event: + +```javascript +personStream.on('metadata', function (type, codec) { /* Something useful. */ }); +``` + +To access a file's header synchronously, there also exists an +[`extractFileHeader`](Api#extractfileheaderpath-opts) method: + +```javascript +var header = avro.extractFileHeader('persons.avro'); +``` + +Writing to an Avro container file is possible using +[`createFileEncoder`](Api#createfileencoderpath-type-opts): + +```javascript +var encoder = avro.createFileEncoder('./processed.avro', type); +``` + + +## Next steps + +The [API documentation](Api) provides a comprehensive list of available +functions and their options. The [Advanced usage section](Advanced-usage) goes +through a few examples to show how the API can be used. + + + +[object-container]: https://avro.apache.org/docs/current/spec.html#Object+Container+Files +[rstream]: https://nodejs.org/api/stream.html#stream_class_stream_readable diff --git a/lang/js/etc/browser/avro.js b/lang/js/etc/browser/avro.js new file mode 100644 index 00000000000..7abe861ff61 --- /dev/null +++ b/lang/js/etc/browser/avro.js @@ -0,0 +1,91 @@ +/* jshint browserify: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +/** + * Shim entry point used when `avro` is `require`d from browserify. + * + * It doesn't expose any of the filesystem methods and patches a few others. + * + */ + +var Tap = require('../../lib/utils').Tap, + schemas = require('../../lib/schemas'), + deprecated = require('../deprecated/validator'); + + +function parse(schema, opts) { + var obj; + if (typeof schema == 'string') { + try { + obj = JSON.parse(schema); + } catch (err) { + // Pass. No file reading from the browser. + } + } + if (obj === undefined) { + obj = schema; + } + return schemas.createType(obj, opts); +} + +// No utf8 and binary functions on browserify's `Buffer`, we must patch in the +// generic slice and write equivalents. + +Tap.prototype.readString = function () { + var len = this.readLong(); + var pos = this.pos; + var buf = this.buf; + this.pos += len; + if (this.pos > buf.length) { + return; + } + return this.buf.slice(pos, pos + len).toString(); +}; + +Tap.prototype.writeString = function (s) { + var len = Buffer.byteLength(s); + this.writeLong(len); + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + this.buf.write(s, pos); +}; + +Tap.prototype.writeBinary = function (s, len) { + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + this.buf.write(s, pos, len, 'binary'); +}; + + +module.exports = { + parse: parse, + types: schemas.types, + Validator: deprecated.Validator, + ProtocolValidator: deprecated.ProtocolValidator +}; diff --git a/lang/js/etc/browser/crypto.js b/lang/js/etc/browser/crypto.js new file mode 100644 index 00000000000..195bd817682 --- /dev/null +++ b/lang/js/etc/browser/crypto.js @@ -0,0 +1,36 @@ +/* jshint browserify: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +/** + * Shim to disable schema fingerprint computation. + * + */ + +function createHash() { + throw new Error('fingerprinting not supported in the browser'); +} + + +module.exports = { + createHash: createHash +}; diff --git a/lang/js/Gruntfile.js b/lang/js/etc/deprecated/Gruntfile.js similarity index 100% rename from lang/js/Gruntfile.js rename to lang/js/etc/deprecated/Gruntfile.js diff --git a/lang/js/README b/lang/js/etc/deprecated/README similarity index 100% rename from lang/js/README rename to lang/js/etc/deprecated/README diff --git a/lang/js/test/validator.js b/lang/js/etc/deprecated/test_validator.js similarity index 99% rename from lang/js/test/validator.js rename to lang/js/etc/deprecated/test_validator.js index b05e93d536c..2c11da28f89 100644 --- a/lang/js/test/validator.js +++ b/lang/js/etc/deprecated/test_validator.js @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -var validator = require('../lib/validator.js'); +var validator = require('./validator'); var Validator = validator.Validator; var ProtocolValidator = validator.ProtocolValidator; diff --git a/lang/js/lib/validator.js b/lang/js/etc/deprecated/validator.js similarity index 98% rename from lang/js/lib/validator.js rename to lang/js/etc/deprecated/validator.js index bda44a78cc6..3e47305a5e1 100644 --- a/lang/js/lib/validator.js +++ b/lang/js/etc/deprecated/validator.js @@ -13,9 +13,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -if (typeof require !== 'undefined') { - var _ = require("underscore"); -} +var _ = require("underscore"), + util = require('util'); + +var WARNING = 'Validator API is deprecated. Please use the type API instead.'; +Validator = util.deprecate(Validator, WARNING); +ProtocolValidator = util.deprecate(ProtocolValidator, WARNING); var AvroSpec = { PrimitiveTypes: ['null', 'boolean', 'int', 'long', 'float', 'double', 'bytes', 'string'], @@ -409,7 +412,7 @@ function Validator(schema, namespace, namedTypes) { Validator.validate = function(schema, obj) { return (new Validator(schema)).validate(obj); -}; +} function ProtocolValidator(protocol) { this.validate = function(typeName, obj) { diff --git a/lang/js/lib/files.js b/lang/js/lib/files.js new file mode 100644 index 00000000000..b9c6da0a645 --- /dev/null +++ b/lang/js/lib/files.js @@ -0,0 +1,666 @@ +/* jshint node: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +var schemas = require('./schemas'), + utils = require('./utils'), + fs = require('fs'), + stream = require('stream'), + util = require('util'), + zlib = require('zlib'); + +// Type of Avro header. +var HEADER_TYPE = schemas.createType({ + type: 'record', + name: 'org.apache.avro.file.Header', + fields : [ + {name: 'magic', type: {type: 'fixed', name: 'Magic', size: 4}}, + {name: 'meta', type: {type: 'map', values: 'bytes'}}, + {name: 'sync', type: {type: 'fixed', name: 'Sync', size: 16}} + ] +}); + +// Type of each block. +var BLOCK_TYPE = schemas.createType({ + type: 'record', + name: 'org.apache.avro.file.Block', + fields : [ + {name: 'count', type: 'long'}, + {name: 'data', type: 'bytes'}, + {name: 'sync', type: {type: 'fixed', name: 'Sync', size: 16}} + ] +}); + +// Used to toBuffer each block, without having to copy all its data. +var LONG_TYPE = schemas.createType('long'); + +// First 4 bytes of an Avro object container file. +var MAGIC_BYTES = new Buffer('Obj\x01'); + +// Convenience. +var f = util.format; +var Tap = utils.Tap; + + +/** + * Parse a schema and return the corresponding type. + * + */ +function parse(schema, opts) { + return schemas.createType(loadSchema(schema), opts); +} + + +/** + * Duplex stream for decoding fragments. + * + */ +function RawDecoder(schema, opts) { + opts = opts || {}; + + var decode = opts.decode === undefined ? true : !!opts.decode; + stream.Duplex.call(this, { + readableObjectMode: decode, + allowHalfOpen: false + }); + // Somehow setting this to false only closes the writable side after the + // readable side ends, while we need the other way. So we do it manually. + + this._type = parse(schema); + this._tap = new Tap(new Buffer(0)); + this._needPush = false; + this._readValue = createReader(decode, this._type); + this._finished = false; + + this.on('finish', function () { + this._finished = true; + this._read(); + }); +} +util.inherits(RawDecoder, stream.Duplex); + +RawDecoder.prototype._write = function (chunk, encoding, cb) { + var tap = this._tap; + tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]); + tap.pos = 0; + if (this._needPush) { + this._needPush = false; + this._read(); + } + cb(); +}; + +RawDecoder.prototype._read = function () { + var tap = this._tap; + var pos = tap.pos; + var val = this._readValue(tap); + if (tap.isValid()) { + this.push(val); + } else if (!this._finished) { + tap.pos = pos; + this._needPush = true; + } else { + this.push(null); + } +}; + + +/** + * Duplex stream for decoding object container files. + * + */ +function BlockDecoder(opts) { + opts = opts || {}; + + var decode = opts.decode === undefined ? true : !!opts.decode; + stream.Duplex.call(this, { + allowHalfOpen: true, // For async decompressors. + readableObjectMode: decode + }); + + this._type = null; + this._codecs = opts.codecs; + this._parseOpts = opts.parseOpts || {}; + this._tap = new Tap(new Buffer(0)); + this._blockTap = new Tap(new Buffer(0)); + this._syncMarker = null; + this._readValue = null; + this._decode = decode; + this._queue = new utils.OrderedQueue(); + this._decompress = null; // Decompression function. + this._index = 0; // Next block index. + this._pending = 0; // Number of blocks undergoing decompression. + this._needPush = false; + this._finished = false; + + this.on('finish', function () { + this._finished = true; + if (!this._pending) { + this.push(null); + } + }); +} +util.inherits(BlockDecoder, stream.Duplex); + +BlockDecoder.getDefaultCodecs = function () { + return { + 'null': function (buf, cb) { cb(null, buf); }, + 'deflate': zlib.inflateRaw + }; +}; + +BlockDecoder.prototype._decodeHeader = function () { + var tap = this._tap; + var header = HEADER_TYPE._read(tap); + if (!tap.isValid()) { + // Wait until more data arrives. + return false; + } + + if (!MAGIC_BYTES.equals(header.magic)) { + this.emit('error', new Error('invalid magic bytes')); + return; + } + + var codec = (header.meta['avro.codec'] || 'null').toString(); + this._decompress = (this._codecs || BlockDecoder.getDefaultCodecs())[codec]; + if (!this._decompress) { + this.emit('error', new Error(f('unknown codec: %s', codec))); + return; + } + + try { + var schema = JSON.parse(header.meta['avro.schema'].toString()); + this._type = parse(schema, this._parseOpts); + } catch (err) { + this.emit('error', err); + return; + } + + this._readValue = createReader(this._decode, this._type); + this._syncMarker = header.sync; + this.emit('metadata', this._type, codec, header); + return true; +}; + +BlockDecoder.prototype._write = function (chunk, encoding, cb) { + var tap = this._tap; + tap.buf = Buffer.concat([tap.buf, chunk]); + tap.pos = 0; + + if (!this._decodeHeader()) { + process.nextTick(cb); + return; + } + + // We got the header, switch to block decoding mode. Also, call it directly + // in case we already have all the data (in which case `_write` wouldn't get + // called anymore). + this._write = this._writeChunk; + this._write(new Buffer(0), encoding, cb); +}; + +BlockDecoder.prototype._writeChunk = function (chunk, encoding, cb) { + var tap = this._tap; + tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]); + tap.pos = 0; + + var block; + while ((block = tryReadBlock(tap))) { + if (!this._syncMarker.equals(block.sync)) { + cb(new Error('invalid sync marker')); + return; + } + this._decompress(block.data, this._createBlockCallback()); + } + + cb(); +}; + +BlockDecoder.prototype._createBlockCallback = function () { + var self = this; + var index = this._index++; + this._pending++; + + return function (err, data) { + if (err) { + self.emit('error', err); + return; + } + self._pending--; + self._queue.push(new BlockData(index, data)); + if (self._needPush) { + self._needPush = false; + self._read(); + } + }; +}; + +BlockDecoder.prototype._read = function () { + var tap = this._blockTap; + if (tap.pos >= tap.buf.length) { + var data = this._queue.pop(); + if (!data) { + if (this._finished && !this._pending) { + this.push(null); + } else { + this._needPush = true; + } + return; // Wait for more data. + } + tap.buf = data.buf; + tap.pos = 0; + } + + this.push(this._readValue(tap)); // The read is guaranteed valid. +}; + + +/** + * Duplex stream for encoding. + * + */ +function RawEncoder(schema, opts) { + opts = opts || {}; + + stream.Transform.call(this, { + writableObjectMode: true, + allowHalfOpen: false + }); + + this._type = parse(schema); + this._writeValue = function (tap, val) { + try { + this._type._write(tap, val); + } catch (err) { + this.emit('error', err); + } + }; + this._tap = new Tap(new Buffer(opts.batchSize || 65536)); +} +util.inherits(RawEncoder, stream.Transform); + +RawEncoder.prototype._transform = function (val, encoding, cb) { + var tap = this._tap; + var buf = tap.buf; + var pos = tap.pos; + + this._writeValue(tap, val); + if (!tap.isValid()) { + if (pos) { + // Emit any valid data. + this.push(copyBuffer(tap.buf, 0, pos)); + } + var len = tap.pos - pos; + if (len > buf.length) { + // Not enough space for last written object, need to resize. + tap.buf = new Buffer(2 * len); + } + tap.pos = 0; + this._writeValue(tap, val); // Rewrite last failed write. + } + + cb(); +}; + +RawEncoder.prototype._flush = function (cb) { + var tap = this._tap; + var pos = tap.pos; + if (pos) { + // This should only ever be false if nothing is written to the stream. + this.push(tap.buf.slice(0, pos)); + } + cb(); +}; + + +/** + * Duplex stream to write object container files. + * + * @param schema + * @param opts {Object} + * + * + `blockSize`, uncompressed. + * + `codec` + * + `codecs` + * + `noCheck` + * + `omitHeader`, useful to append to an existing block file. + * + */ +function BlockEncoder(schema, opts) { + opts = opts || {}; + + stream.Duplex.call(this, { + allowHalfOpen: true, // To support async compressors. + writableObjectMode: true + }); + + var obj, type; + if (schema instanceof schemas.types.Type) { + type = schema; + schema = undefined; + } else { + // Keep full schema to be able to write it to the header later. + obj = loadSchema(schema); + type = schemas.createType(obj); + schema = JSON.stringify(obj); + } + + this._schema = schema; + this._type = type; + this._writeValue = function (tap, val) { + try { + this._type._write(tap, val); + } catch (err) { + this.emit('error', err); + } + }; + this._blockSize = opts.blockSize || 65536; + this._tap = new Tap(new Buffer(this._blockSize)); + this._codecs = opts.codecs; + this._codec = opts.codec || 'null'; + this._compress = null; + this._omitHeader = opts.omitHeader || false; + this._blockCount = 0; + this._syncMarker = opts.syncMarker || new utils.Lcg().nextBuffer(16); + this._queue = new utils.OrderedQueue(); + this._pending = 0; + this._finished = false; + this._needPush = false; + + this.on('finish', function () { + this._finished = true; + if (this._blockCount) { + this._flushChunk(); + } + }); +} +util.inherits(BlockEncoder, stream.Duplex); + +BlockEncoder.getDefaultCodecs = function () { + return { + 'null': function (buf, cb) { cb(null, buf); }, + 'deflate': zlib.deflateRaw + }; +}; + +BlockEncoder.prototype._write = function (val, encoding, cb) { + var codec = this._codec; + this._compress = (this._codecs || BlockEncoder.getDefaultCodecs())[codec]; + if (!this._compress) { + this.emit('error', new Error(f('unsupported codec: %s', codec))); + return; + } + + if (!this._omitHeader) { + var meta = { + 'avro.schema': new Buffer(this._schema || this._type.getSchema()), + 'avro.codec': new Buffer(this._codec) + }; + var Header = HEADER_TYPE.getRecordConstructor(); + var header = new Header(MAGIC_BYTES, meta, this._syncMarker); + this.push(header.$toBuffer()); + } + + this._write = this._writeChunk; + this._write(val, encoding, cb); +}; + +BlockEncoder.prototype._writeChunk = function (val, encoding, cb) { + var tap = this._tap; + var pos = tap.pos; + + this._writeValue(tap, val); + if (!tap.isValid()) { + if (pos) { + this._flushChunk(pos); + } + var len = tap.pos - pos; + if (len > this._blockSize) { + // Not enough space for last written object, need to resize. + this._blockSize = len * 2; + } + tap.buf = new Buffer(this._blockSize); + tap.pos = 0; + this._writeValue(tap, val); // Rewrite last failed write. + } + this._blockCount++; + + cb(); +}; + +BlockEncoder.prototype._flushChunk = function (pos) { + var tap = this._tap; + pos = pos || tap.pos; + this._compress(tap.buf.slice(0, pos), this._createBlockCallback()); + this._blockCount = 0; +}; + +BlockEncoder.prototype._read = function () { + var self = this; + var data = this._queue.pop(); + if (!data) { + if (this._finished && !this._pending) { + process.nextTick(function () { self.push(null); }); + } else { + this._needPush = true; + } + return; + } + + this.push(LONG_TYPE.toBuffer(data.count, true)); + this.push(LONG_TYPE.toBuffer(data.buf.length, true)); + this.push(data.buf); + this.push(this._syncMarker); +}; + +BlockEncoder.prototype._createBlockCallback = function () { + var self = this; + var index = this._index++; + var count = this._blockCount; + this._pending++; + + return function (err, data) { + if (err) { + self.emit('error', err); + return; + } + self._pending--; + self._queue.push(new BlockData(index, data, count)); + if (self._needPush) { + self._needPush = false; + self._read(); + } + }; +}; + + +/** + * Extract a container file's header synchronously. + * + */ +function extractFileHeader(path, opts) { + opts = opts || {}; + + var decode = opts.decode === undefined ? true : !!opts.decode; + var size = Math.max(opts.size || 4096, 4); + var fd = fs.openSync(path, 'r'); + var buf = new Buffer(size); + var pos = 0; + var tap = new Tap(buf); + var header = null; + + while (pos < 4) { + // Make sure we have enough to check the magic bytes. + pos += fs.readSync(fd, buf, pos, size - pos); + } + if (MAGIC_BYTES.equals(buf.slice(0, 4))) { + do { + header = HEADER_TYPE._read(tap); + } while (!isValid()); + if (decode !== false) { + var meta = header.meta; + meta['avro.schema'] = JSON.parse(meta['avro.schema'].toString()); + if (meta['avro.codec'] !== undefined) { + meta['avro.codec'] = meta['avro.codec'].toString(); + } + } + } + fs.closeSync(fd); + return header; + + function isValid() { + if (tap.isValid()) { + return true; + } + var len = 2 * tap.buf.length; + var buf = new Buffer(len); + len = fs.readSync(fd, buf, 0, len); + tap.buf = Buffer.concat([tap.buf, buf]); + tap.pos = 0; + return false; + } +} + + +/** + * Readable stream of records from a local Avro file. + * + */ +function createFileDecoder(path, opts) { + return fs.createReadStream(path).pipe(new BlockDecoder(opts)); +} + + +/** + * Writable stream of records to a local Avro file. + * + */ +function createFileEncoder(path, schema, opts) { + var encoder = new BlockEncoder(schema, opts); + encoder.pipe(fs.createWriteStream(path, {defaultEncoding: 'binary'})); + return encoder; +} + + +// Helpers. + +/** + * An indexed block. + * + * This can be used to preserve block order since compression and decompression + * can cause some some blocks to be returned out of order. The count is only + * used when encoding. + * + */ +function BlockData(index, buf, count) { + this.index = index; + this.buf = buf; + this.count = count | 0; +} + +/** + * Maybe get a block. + * + */ +function tryReadBlock(tap) { + var pos = tap.pos; + var block = BLOCK_TYPE._read(tap); + if (!tap.isValid()) { + tap.pos = pos; + return null; + } + return block; +} + +/** + * Create bytes consumer, either reading or skipping records. + * + */ +function createReader(decode, type) { + if (decode) { + return function (tap) { return type._read(tap); }; + } else { + return (function (skipper) { + return function (tap) { + var pos = tap.pos; + skipper(tap); + return tap.buf.slice(pos, tap.pos); + }; + })(type._skip); + } +} + +/** + * Copy a buffer. + * + * This avoids having to create a slice of the original buffer. + * + */ +function copyBuffer(buf, pos, len) { + var copy = new Buffer(len); + buf.copy(copy, 0, pos, pos + len); + return copy; +} + +/** + * Try to load a schema. + * + * This method will attempt to load schemas from a file if the schema passed is + * a string which isn't valid JSON and contains at least one slash. + * + */ +function loadSchema(schema) { + var obj; + if (typeof schema == 'string') { + try { + obj = JSON.parse(schema); + } catch (err) { + if (~schema.indexOf('/')) { + // This can't be a valid name, so we interpret is as a filepath. This + // makes is always feasible to read a file, independent of its name + // (i.e. even if its name is valid JSON), by prefixing it with `./`. + obj = JSON.parse(fs.readFileSync(schema)); + } + } + } + if (obj === undefined) { + obj = schema; + } + return obj; +} + + +module.exports = { + HEADER_TYPE: HEADER_TYPE, // For tests. + MAGIC_BYTES: MAGIC_BYTES, // Idem. + parse: parse, + createFileDecoder: createFileDecoder, + createFileEncoder: createFileEncoder, + extractFileHeader: extractFileHeader, + streams: { + RawDecoder: RawDecoder, + BlockDecoder: BlockDecoder, + RawEncoder: RawEncoder, + BlockEncoder: BlockEncoder + } +}; diff --git a/lang/js/lib/index.js b/lang/js/lib/index.js new file mode 100644 index 00000000000..0eab2ad74e4 --- /dev/null +++ b/lang/js/lib/index.js @@ -0,0 +1,45 @@ +/* jshint node: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +/** + * Main node.js entry point. + * + * See `etc/browser/avro.js` for the entry point used for browserify. + * + */ + +var files = require('./files'), + schemas = require('./schemas'), + deprecated = require('../etc/deprecated/validator'); + + +module.exports = { + parse: files.parse, + createFileDecoder: files.createFileDecoder, + createFileEncoder: files.createFileEncoder, + extractFileHeader: files.extractFileHeader, + streams: files.streams, + types: schemas.types, + Validator: deprecated.Validator, + ProtocolValidator: deprecated.ProtocolValidator +}; diff --git a/lang/js/lib/schemas.js b/lang/js/lib/schemas.js new file mode 100644 index 00000000000..1ca9d6947cf --- /dev/null +++ b/lang/js/lib/schemas.js @@ -0,0 +1,2197 @@ +/* jshint node: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +var utils = require('./utils'), + buffer = require('buffer'), // For `SlowBuffer`. + crypto = require('crypto'), + util = require('util'); + +// Convenience imports. +var Tap = utils.Tap; +var f = util.format; + +// All Avro types. +var TYPES = { + 'array': ArrayType, + 'boolean': BooleanType, + 'bytes': BytesType, + 'double': DoubleType, + 'enum': EnumType, + 'error': RecordType, + 'fixed': FixedType, + 'float': FloatType, + 'int': IntType, + 'long': LongType, + 'map': MapType, + 'null': NullType, + 'record': RecordType, + 'string': StringType, + 'union': UnionType +}; + +// Valid (field, type, and symbol) name regex. +var NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +// Random generator. +var RANDOM = new utils.Lcg(); + +// Encoding tap (shared for performance). +var TAP = new Tap(new buffer.SlowBuffer(1024)); + +// Path prefix for validity checks (shared for performance). +var PATH = []; + +// Currently active logical type, used for name redirection. +var LOGICAL_TYPE = null; + + +/** + * Schema parsing entry point. + * + * It isn't exposed directly but called from `parse` inside `index.js` (node) + * or `avro.js` (browserify) which each add convenience functionality. + * + */ +function createType(attrs, opts) { + if (attrs instanceof Type) { + return attrs; + } + + opts = getOpts(attrs, opts); + + var type; + if (typeof attrs == 'string') { // Type reference. + if (opts.namespace && !~attrs.indexOf('.') && !isPrimitive(attrs)) { + attrs = opts.namespace + '.' + attrs; + } + type = opts.registry[attrs]; + if (type) { + // Type was already defined, return it. + return type; + } + if (isPrimitive(attrs)) { + // Reference to a primitive type. These are also defined names by default + // so we create the appropriate type and it to the registry for future + // reference. + return opts.registry[attrs] = createType({type: attrs}, opts); + } + throw new Error(f('undefined type name: %s', attrs)); + } + + if (opts.typeHook && (type = opts.typeHook(attrs, opts))) { + if (!(type instanceof Type)) { + throw new Error(f('invalid typehook return value: %j', type)); + } + return type; + } + + if (attrs.logicalType && !LOGICAL_TYPE) { + var DerivedType = opts.logicalTypes[attrs.logicalType]; + if (DerivedType) { + var registry = {}; + Object.keys(opts.registry).forEach(function (key) { + registry[key] = opts.registry[key]; + }); + try { + return new DerivedType(attrs, opts); + } catch (err) { + if (opts.assertLogicalType) { + // The spec mandates that we fall through to the underlying type if + // the logical type is invalid. We provide this option to ease + // debugging. + throw err; + } + LOGICAL_TYPE = null; + opts.registry = registry; // In case any names were registered. + } + } + } + + if (attrs instanceof Array) { // Union. + type = new UnionType(attrs, opts); + } else { // New type definition. + type = (function (typeName) { + var Type = TYPES[typeName]; + if (Type === undefined) { + throw new Error(f('unknown type: %j', typeName)); + } + return new Type(attrs, opts); + })(attrs.type); + } + return type; +} + +/** + * "Abstract" base Avro type class. + * + * This class' constructor will register any named types to support + * recursive schemas. + * + * All type values are represented in memory similarly to their JSON + * representation, except for `bytes` and `fixed` which are represented as + * `Buffer`s. See individual subclasses for details. + * + */ +function Type(registry) { + var name = this._name; + var type = LOGICAL_TYPE || this; + LOGICAL_TYPE = null; + + if (registry === undefined || name === undefined) { + return; + } + + var prev = registry[name]; + if (prev !== undefined) { + throw new Error(f('duplicate type name: %s', name)); + } + registry[name] = type; +} + +Type.__reset = function (size) { TAP.buf = new buffer.SlowBuffer(size); }; + +Type.prototype.createResolver = function (type, opts) { + if (!(type instanceof Type)) { + // More explicit error message than the "incompatible type" thrown + // otherwise (especially because of the overridden `toJSON` method). + throw new Error(f('not a type: %j', type)); + } + + if (type instanceof LogicalType && !(this instanceof LogicalType)) { + // Trying to read a logical type as a built-in: unwrap the logical type. + return this.createResolver(type._underlyingType, opts); + } + + opts = opts || {}; + opts.registry = opts.registry || {}; + + var resolver, key; + if (this instanceof RecordType && type instanceof RecordType) { + key = this._name + ':' + type._name; // ':' is illegal in Avro type names. + resolver = opts.registry[key]; + if (resolver) { + return resolver; + } + } + + resolver = new Resolver(this); + if (key) { // Register resolver early for recursive schemas. + opts.registry[key] = resolver; + } + + if (type instanceof UnionType) { + var resolvers = type._types.map(function (t) { + return this.createResolver(t, opts); + }, this); + resolver._read = function (tap) { + var index = tap.readLong(); + var resolver = resolvers[index]; + if (resolver === undefined) { + throw new Error(f('invalid union index: %s', index)); + } + return resolvers[index]._read(tap); + }; + } else { + this._updateResolver(resolver, type, opts); + } + + if (!resolver._read) { + throw new Error(f('cannot read %s as %s', type, this)); + } + return resolver; +}; + +Type.prototype.decode = function (buf, pos, resolver) { + var tap = new Tap(buf); + tap.pos = pos | 0; + var val = readValue(this, tap, resolver); + if (!tap.isValid()) { + return {value: undefined, offset: -1}; + } + return {value: val, offset: tap.pos}; +}; + +Type.prototype.encode = function (val, buf, pos) { + var tap = new Tap(buf); + tap.pos = pos | 0; + this._write(tap, val); + if (!tap.isValid()) { + // Don't throw as there is no way to predict this. We also return the + // number of missing bytes to ease resizing. + return buf.length - tap.pos; + } + return tap.pos; +}; + +Type.prototype.fromBuffer = function (buf, resolver, noCheck) { + var tap = new Tap(buf); + var val = readValue(this, tap, resolver, noCheck); + if (!tap.isValid()) { + throw new Error('truncated buffer'); + } + if (!noCheck && tap.pos < buf.length) { + throw new Error('trailing data'); + } + return val; +}; + +Type.prototype.toBuffer = function (val) { + TAP.pos = 0; + this._write(TAP, val); + if (!TAP.isValid()) { + Type.__reset(2 * TAP.pos); + TAP.pos = 0; + this._write(TAP, val); + } + var buf = new Buffer(TAP.pos); + TAP.buf.copy(buf, 0, 0, TAP.pos); + return buf; +}; + +Type.prototype.fromString = function (str) { + return this._copy(JSON.parse(str), {coerce: 2}); +}; + +Type.prototype.toString = function (val) { + if (val === undefined) { + // Consistent behavior with standard `toString` expectations. + return this.getSchema(true); + } + return JSON.stringify(this._copy(val, {coerce: 3})); +}; + +Type.prototype.clone = function (val, opts) { + if (opts) { + opts = { + coerce: !!opts.coerceBuffers | 0, // Coerce JSON to Buffer. + fieldHook: opts.fieldHook, + wrap: !!opts.wrapUnions | 0 // Wrap first match into union. + }; + } + return this._copy(val, opts); +}; + +Type.prototype.isValid = function (val, opts) { + while (PATH.length) { + // In case the previous `isValid` call didn't complete successfully (e.g. + // if an exception was thrown, but then caught in client code), `PATH` + // might be non-empty, we must manually clear it. + PATH.pop(); + } + return this._check(val, opts && opts.errorHook); +}; + +Type.prototype.compareBuffers = function (buf1, buf2) { + return this._match(new Tap(buf1), new Tap(buf2)); +}; + +Type.prototype.getName = function () { return this._name; }; + +Type.prototype.getSchema = function (noDeref) { + // Since JS objects are unordered, this implementation (unfortunately) + // relies on engines returning properties in the same order that they are + // inserted in. This is not in the JS spec, but can be "somewhat" safely + // assumed (more here: http://stackoverflow.com/q/5525795/1062617). + return (function (type, registry) { + return JSON.stringify(type, function (key, value) { + if (value instanceof Field) { + return {name: value._name, type: value._type}; + } else if (value && value.name) { + var name = value.name; + if (noDeref || registry[name]) { + return name; + } + registry[name] = true; + } + return value; + }); + })(this, {}); +}; + +Type.prototype.getFingerprint = function (algorithm) { + algorithm = algorithm || 'md5'; + var hash = crypto.createHash(algorithm); + hash.end(this.getSchema()); + return hash.read(); +}; + +Type.prototype.inspect = function () { + if (this instanceof PrimitiveType) { + return f('<%s>', this.constructor.name); + } else { + var obj = JSON.parse(this.getSchema(true)); // Slow, only for debugging. + if (typeof obj == 'object') { + obj.type = undefined; // Would be redundant with constructor name. + } + return f('<%s %j>', this.constructor.name, obj); + } +}; + +Type.prototype._check = utils.abstractFunction; +Type.prototype._copy = utils.abstractFunction; +Type.prototype._match = utils.abstractFunction; +Type.prototype._read = utils.abstractFunction; +Type.prototype._skip = utils.abstractFunction; +Type.prototype._updateResolver = utils.abstractFunction; +Type.prototype._write = utils.abstractFunction; +Type.prototype.compare = utils.abstractFunction; +Type.prototype.random = utils.abstractFunction; + +// Implementations. + +/** + * Base primitive Avro type. + * + * Most of the primitive types share the same cloning and resolution + * mechanisms, provided by this class. This class also lets us conveniently + * check whether a type is a primitive using `instanceof`. + * + */ +function PrimitiveType() { Type.call(this); } +util.inherits(PrimitiveType, Type); +PrimitiveType.prototype._updateResolver = function (resolver, type) { + if (type.constructor === this.constructor) { + resolver._read = this._read; + } +}; +PrimitiveType.prototype._copy = function (val) { + this._check(val, throwInvalidError); + return val; +}; +PrimitiveType.prototype.compare = utils.compare; + +/** + * Nulls. + * + */ +function NullType() { PrimitiveType.call(this); } +util.inherits(NullType, PrimitiveType); +NullType.prototype._check = function (val, cb) { + var b = val === null; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +NullType.prototype._read = function () { return null; }; +NullType.prototype._skip = function () {}; +NullType.prototype._write = function (tap, val) { + if (val !== null) { + throwInvalidError(null, val, this); + } +}; +NullType.prototype._match = function () { return 0; }; +NullType.prototype.compare = NullType.prototype._match; +NullType.prototype.random = NullType.prototype._read; +NullType.prototype.toJSON = function () { return 'null'; }; + +/** + * Booleans. + * + */ +function BooleanType() { PrimitiveType.call(this); } +util.inherits(BooleanType, PrimitiveType); +BooleanType.prototype._check = function (val, cb) { + var b = typeof val == 'boolean'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +BooleanType.prototype._read = function (tap) { return tap.readBoolean(); }; +BooleanType.prototype._skip = function (tap) { tap.skipBoolean(); }; +BooleanType.prototype._write = function (tap, val) { + if (typeof val != 'boolean') { + throwInvalidError(null, val, this); + } + tap.writeBoolean(val); +}; +BooleanType.prototype._match = function (tap1, tap2) { + return tap1.matchBoolean(tap2); +}; +BooleanType.prototype.random = function () { return RANDOM.nextBoolean(); }; +BooleanType.prototype.toJSON = function () { return 'boolean'; }; + +/** + * Integers. + * + */ +function IntType() { PrimitiveType.call(this); } +util.inherits(IntType, PrimitiveType); +IntType.prototype._check = function (val, cb) { + var b = val === (val | 0); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +IntType.prototype._read = function (tap) { return tap.readInt(); }; +IntType.prototype._skip = function (tap) { tap.skipInt(); }; +IntType.prototype._write = function (tap, val) { + if (val !== (val | 0)) { + throwInvalidError(null, val, this); + } + tap.writeInt(val); +}; +IntType.prototype._match = function (tap1, tap2) { + return tap1.matchInt(tap2); +}; +IntType.prototype.random = function () { return RANDOM.nextInt(1000) | 0; }; +IntType.prototype.toJSON = function () { return 'int'; }; + +/** + * Longs. + * + * We can't capture all the range unfortunately since JavaScript represents all + * numbers internally as `double`s, so the default implementation plays safe + * and throws rather than potentially silently change the data. See `using` or + * `AbstractLongType` below for a way to implement a custom long type. + * + */ +function LongType() { PrimitiveType.call(this); } +util.inherits(LongType, PrimitiveType); +LongType.prototype._check = function (val, cb) { + var b = typeof val == 'number' && val % 1 === 0 && isSafeLong(val); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +LongType.prototype._read = function (tap) { + var n = tap.readLong(); + if (!isSafeLong(n)) { + throw new Error('potential precision loss'); + } + return n; +}; +LongType.prototype._skip = function (tap) { tap.skipLong(); }; +LongType.prototype._write = function (tap, val) { + if (typeof val != 'number' || val % 1 || !isSafeLong(val)) { + throwInvalidError(null, val, this); + } + tap.writeLong(val); +}; +LongType.prototype._match = function (tap1, tap2) { + return tap1.matchLong(tap2); +}; +LongType.prototype._updateResolver = function (resolver, type) { + if (type instanceof LongType || type instanceof IntType) { + resolver._read = type._read; + } +}; +LongType.prototype.random = function () { return RANDOM.nextInt(); }; +LongType.prototype.toJSON = function () { return 'long'; }; +LongType.using = function (methods, noUnpack) { + methods = methods || {}; // Will give a more helpful error message. + // We map some of the methods to a different name to be able to intercept + // their input and output (otherwise we wouldn't be able to perform any + // unpacking logic, and the type wouldn't work when nested). + var mapping = { + toBuffer: '_toBuffer', + fromBuffer: '_fromBuffer', + fromJSON: '_fromJSON', + toJSON: '_toJSON', + isValid: '_isValid', + compare: 'compare' + }; + var type = new AbstractLongType(noUnpack); + Object.keys(mapping).forEach(function (name) { + if (methods[name] === undefined) { + throw new Error(f('missing method implementation: %s', name)); + } + type[mapping[name]] = methods[name]; + }); + return type; +}; + +/** + * Floats. + * + */ +function FloatType() { PrimitiveType.call(this); } +util.inherits(FloatType, PrimitiveType); +FloatType.prototype._check = function (val, cb) { + var b = typeof val == 'number'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +FloatType.prototype._read = function (tap) { return tap.readFloat(); }; +FloatType.prototype._skip = function (tap) { tap.skipFloat(); }; +FloatType.prototype._write = function (tap, val) { + if (typeof val != 'number') { + throwInvalidError(null, val, this); + } + tap.writeFloat(val); +}; +FloatType.prototype._match = function (tap1, tap2) { + return tap1.matchFloat(tap2); +}; +FloatType.prototype._updateResolver = function (resolver, type) { + if ( + type instanceof FloatType || + type instanceof LongType || + type instanceof IntType + ) { + resolver._read = type._read; + } +}; +FloatType.prototype.random = function () { return RANDOM.nextFloat(1e3); }; +FloatType.prototype.toJSON = function () { return 'float'; }; + +/** + * Doubles. + * + */ +function DoubleType() { PrimitiveType.call(this); } +util.inherits(DoubleType, PrimitiveType); +DoubleType.prototype._check = function (val, cb) { + var b = typeof val == 'number'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +DoubleType.prototype._read = function (tap) { return tap.readDouble(); }; +DoubleType.prototype._skip = function (tap) { tap.skipDouble(); }; +DoubleType.prototype._write = function (tap, val) { + if (typeof val != 'number') { + throwInvalidError(null, val, this); + } + tap.writeDouble(val); +}; +DoubleType.prototype._match = function (tap1, tap2) { + return tap1.matchDouble(tap2); +}; +DoubleType.prototype._updateResolver = function (resolver, type) { + if ( + type instanceof DoubleType || + type instanceof FloatType || + type instanceof LongType || + type instanceof IntType + ) { + resolver._read = type._read; + } +}; +DoubleType.prototype.random = function () { return RANDOM.nextFloat(); }; +DoubleType.prototype.toJSON = function () { return 'double'; }; + +/** + * Strings. + * + */ +function StringType() { PrimitiveType.call(this); } +util.inherits(StringType, PrimitiveType); +StringType.prototype._check = function (val, cb) { + var b = typeof val == 'string'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +StringType.prototype._read = function (tap) { return tap.readString(); }; +StringType.prototype._skip = function (tap) { tap.skipString(); }; +StringType.prototype._write = function (tap, val) { + if (typeof val != 'string') { + throwInvalidError(null, val, this); + } + tap.writeString(val); +}; +StringType.prototype._match = function (tap1, tap2) { + return tap1.matchString(tap2); +}; +StringType.prototype._updateResolver = function (resolver, type) { + if (type instanceof StringType || type instanceof BytesType) { + resolver._read = this._read; + } +}; +StringType.prototype.random = function () { + return RANDOM.nextString(RANDOM.nextInt(32)); +}; +StringType.prototype.toJSON = function () { return 'string'; }; + +/** + * Bytes. + * + * These are represented in memory as `Buffer`s rather than binary-encoded + * strings. This is more efficient (when decoding/encoding from bytes, the + * common use-case), idiomatic, and convenient. + * + * Note the coercion in `_copy`. + * + */ +function BytesType() { PrimitiveType.call(this); } +util.inherits(BytesType, PrimitiveType); +BytesType.prototype._check = function (val, cb) { + var b = Buffer.isBuffer(val); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +BytesType.prototype._read = function (tap) { return tap.readBytes(); }; +BytesType.prototype._skip = function (tap) { tap.skipBytes(); }; +BytesType.prototype._write = function (tap, val) { + if (!Buffer.isBuffer(val)) { + throwInvalidError(null, val, this); + } + tap.writeBytes(val); +}; +BytesType.prototype._match = function (tap1, tap2) { + return tap1.matchBytes(tap2); +}; +BytesType.prototype._updateResolver = StringType.prototype._updateResolver; +BytesType.prototype._copy = function (obj, opts) { + var buf; + switch ((opts && opts.coerce) | 0) { + case 3: // Coerce buffers to strings. + this._check(obj, throwInvalidError); + return obj.toString('binary'); + case 2: // Coerce strings to buffers. + if (typeof obj != 'string') { + throw new Error(f('cannot coerce to buffer: %j', obj)); + } + buf = new Buffer(obj, 'binary'); + this._check(buf, throwInvalidError); + return buf; + case 1: // Coerce buffer JSON representation to buffers. + if (!obj || obj.type !== 'Buffer' || !(obj.data instanceof Array)) { + throw new Error(f('cannot coerce to buffer: %j', obj)); + } + buf = new Buffer(obj.data); + this._check(buf, throwInvalidError); + return buf; + default: // Copy buffer. + this._check(obj, throwInvalidError); + return new Buffer(obj); + } +}; +BytesType.prototype.compare = Buffer.compare; +BytesType.prototype.random = function () { + return RANDOM.nextBuffer(RANDOM.nextInt(32)); +}; +BytesType.prototype.toJSON = function () { return 'bytes'; }; + +/** + * Avro unions. + * + * Unions are represented in memory similarly to their JSON representation + * (i.e. inside an object with single key the name of the contained type). + * + * This is not ideal, but is the most efficient way to unambiguously support + * all unions. Here are a few reasons why the wrapping object is necessary: + * + * + Unions with multiple number types would have undefined behavior, unless + * numbers are wrapped (either everywhere, leading to large performance and + * convenience costs; or only when necessary inside unions, making it hard to + * understand when numbers are wrapped or not). + * + Fixed types would have to be wrapped to be distinguished from bytes. + * + Using record's constructor names would work (after a slight change to use + * the fully qualified name), but would mean that generic objects could no + * longer be valid records (making it inconvenient to do simple things like + * creating new records). + * + * Lore: In the past (until d304cab), there used to be an "unwrapped union + * type" which directly exposed its values, without the wrapping object + * (similarly to Avro's python implementation). It was removed to keep all + * representations consistent and make this library simpler to understand + * (conversions, e.g. for schema evolution, between representations were + * particularly confusing). Encoding was also much slower (worst case + * complexity linear in the number of types in the union). + * + */ +function UnionType(attrs, opts) { + if (!(attrs instanceof Array)) { + throw new Error(f('non-array union schema: %j', attrs)); + } + if (!attrs.length) { + throw new Error('empty union'); + } + + opts = getOpts(attrs, opts); + Type.call(this); + this._types = attrs.map(function (obj) { return createType(obj, opts); }); + + this._indices = {}; + this._types.forEach(function (type, i) { + if (type instanceof UnionType) { + throw new Error('unions cannot be directly nested'); + } + var name = type._name || getTypeName(type); + if (this._indices[name] !== undefined) { + throw new Error(f('duplicate union name: %j', name)); + } + this._indices[name] = i; + }, this); + + this._constructors = this._types.map(function (type) { + // jshint -W054 + var name = type._name || getTypeName(type); + if (name === 'null') { + return null; + } + var body; + if (~name.indexOf('.')) { // Qualified name. + body = 'this[\'' + name + '\'] = val;'; + } else { + body = 'this.' + name + ' = val;'; + } + return new Function('val', body); + }); +} +util.inherits(UnionType, Type); + +UnionType.prototype._check = function (val, cb) { + var b = false; + if (val === null) { + // Shortcut type lookup in this case. + b = this._indices['null'] !== undefined; + } else if (typeof val == 'object') { + var keys = Object.keys(val); + if (keys.length === 1) { + // We require a single key here to ensure that writes are correct and + // efficient as soon as a record passes this check. + var name = keys[0]; + var index = this._indices[name]; + if (index !== undefined) { + PATH.push(name); + b = this._types[index]._check(val[name], cb); + PATH.pop(); + return b; + } + } + } + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +UnionType.prototype._read = function (tap) { + var index = tap.readLong(); + var Class = this._constructors[index]; + if (Class) { + return new Class(this._types[index]._read(tap)); + } else if (Class === null) { + return null; + } else { + throw new Error(f('invalid union index: %s', index)); + } +}; + +UnionType.prototype._skip = function (tap) { + this._types[tap.readLong()]._skip(tap); +}; + +UnionType.prototype._write = function (tap, val) { + var index, keys, name; + if (val === null) { + index = this._indices['null']; + if (index === undefined) { + throwInvalidError(null, val, this); + } + tap.writeLong(index); + } else { + keys = Object.keys(val); + if (keys.length === 1) { + name = keys[0]; + index = this._indices[name]; + } + if (index === undefined) { + throwInvalidError(null, val, this); + } + tap.writeLong(index); + this._types[index]._write(tap, val[name]); + } +}; + +UnionType.prototype._match = function (tap1, tap2) { + var n1 = tap1.readLong(); + var n2 = tap2.readLong(); + if (n1 === n2) { + return this._types[n1]._match(tap1, tap2); + } else { + return n1 < n2 ? -1 : 1; + } +}; + +UnionType.prototype._updateResolver = function (resolver, type, opts) { + // jshint -W083 + // (The loop exits after the first function is created.) + var i, l, typeResolver, Class; + for (i = 0, l = this._types.length; i < l; i++) { + try { + typeResolver = this._types[i].createResolver(type, opts); + } catch (err) { + continue; + } + Class = this._constructors[i]; + if (Class) { + resolver._read = function (tap) { + return new Class(typeResolver._read(tap)); + }; + } else { + resolver._read = function () { return null; }; + } + return; + } +}; + +UnionType.prototype._copy = function (val, opts) { + var wrap = opts && opts.wrap | 0; + if (wrap === 2) { + // Promote into first type (used for schema defaults). + if (val === null && this._constructors[0] === null) { + return null; + } + return new this._constructors[0](this._types[0]._copy(val, opts)); + } + if (val === null && this._indices['null'] !== undefined) { + return null; + } + var i, l, obj; + if (wrap === 1) { + // Promote into first match (convenience, slow). + i = 0; + l = this._types.length; + while (i < l && obj === undefined) { + try { + obj = this._types[i]._copy(val, opts); + } catch (err) { + i++; + } + } + } else if (typeof val == 'object') { + var keys = Object.keys(val); + if (keys.length === 1) { + var name = keys[0]; + i = this._indices[name]; + if (i === undefined) { + // We are a bit more flexible than in `_check` here since we have + // to deal with other serializers being less strict, so we fall + // back to looking up unqualified names. + var j, type; + for (j = 0, l = this._types.length; j < l; j++) { + type = this._types[j]; + if (type._name && name === unqualify(type._name)) { + i = j; + break; + } + } + } + if (i !== undefined) { + obj = this._types[i]._copy(val[name], opts); + } + } + } + if (obj !== undefined) { + return new this._constructors[i](obj); + } + throwInvalidError(null, val, this); +}; + +UnionType.prototype.compare = function (val1, val2) { + var name1 = val1 === null ? 'null' : Object.keys(val1)[0]; + var name2 = val2 === null ? 'null' : Object.keys(val2)[0]; + var index = this._indices[name1]; + if (name1 === name2) { + return name1 === 'null' ? + 0 : + this._types[index].compare(val1[name1], val2[name1]); + } else { + return utils.compare(index, this._indices[name2]); + } +}; + +UnionType.prototype.getTypes = function () { return this._types.slice(); }; + +UnionType.prototype.random = function () { + var index = RANDOM.nextInt(this._types.length); + var Class = this._constructors[index]; + if (!Class) { + return null; + } + return new Class(this._types[index].random()); +}; + +UnionType.prototype.toJSON = function () { return this._types; }; + +/** + * Avro enum type. + * + * Represented as strings (with allowed values from the set of symbols). Using + * integers would be a reasonable option, but the performance boost is arguably + * offset by the legibility cost and the extra deviation from the JSON encoding + * convention. + * + * An integer representation can still be used (e.g. for compatibility with + * TypeScript `enum`s) by overriding the `EnumType` with a `LongType` (e.g. via + * `parse`'s registry). + * + */ +function EnumType(attrs, opts) { + if (!(attrs.symbols instanceof Array) || !attrs.symbols.length) { + throw new Error(f('invalid %j enum symbols: %j', attrs.name, attrs)); + } + + opts = getOpts(attrs, opts); + var resolutions = resolveNames(attrs, opts.namespace); + this._name = resolutions.name; + this._symbols = attrs.symbols; + this._aliases = resolutions.aliases; + Type.call(this, opts.registry); + + this._indices = {}; + this._symbols.forEach(function (symbol, i) { + if (!NAME_PATTERN.test(symbol)) { + throw new Error(f('invalid %s symbol: %j', this, symbol)); + } + if (this._indices[symbol] !== undefined) { + throw new Error(f('duplicate %s symbol: %j', this, symbol)); + } + this._indices[symbol] = i; + }, this); +} +util.inherits(EnumType, Type); + +EnumType.prototype._check = function (val, cb) { + var b = this._indices[val] !== undefined; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +EnumType.prototype._read = function (tap) { + var index = tap.readLong(); + var symbol = this._symbols[index]; + if (symbol === undefined) { + throw new Error(f('invalid %s enum index: %s', this._name, index)); + } + return symbol; +}; + +EnumType.prototype._skip = function (tap) { tap.skipLong(); }; + +EnumType.prototype._write = function (tap, val) { + var index = this._indices[val]; + if (index === undefined) { + throwInvalidError(null, val, this); + } + tap.writeLong(index); +}; + +EnumType.prototype._match = function (tap1, tap2) { + return tap1.matchLong(tap2); +}; + +EnumType.prototype.compare = function (val1, val2) { + return utils.compare(this._indices[val1], this._indices[val2]); +}; + +EnumType.prototype._updateResolver = function (resolver, type) { + var symbols = this._symbols; + if ( + type instanceof EnumType && + ~getAliases(this).indexOf(type._name) && + type._symbols.every(function (s) { return ~symbols.indexOf(s); }) + ) { + resolver._symbols = type._symbols; + resolver._read = type._read; + } +}; + +EnumType.prototype._copy = function (val) { + this._check(val, throwInvalidError); + return val; +}; + +EnumType.prototype.getAliases = function () { return this._aliases; }; + +EnumType.prototype.getSymbols = function () { return this._symbols.slice(); }; + +EnumType.prototype.random = function () { + return RANDOM.choice(this._symbols); +}; + +EnumType.prototype.toJSON = function () { + return {name: this._name, type: 'enum', symbols: this._symbols}; +}; + +/** + * Avro fixed type. + * + * Represented simply as a `Buffer`. + * + */ +function FixedType(attrs, opts) { + if (attrs.size !== (attrs.size | 0) || attrs.size < 1) { + throw new Error(f('invalid %j fixed size: %j', attrs.name, attrs.size)); + } + + opts = getOpts(attrs, opts); + var resolutions = resolveNames(attrs, opts.namespace); + this._name = resolutions.name; + this._size = attrs.size | 0; + this._aliases = resolutions.aliases; + Type.call(this, opts.registry); +} +util.inherits(FixedType, Type); + +FixedType.prototype._check = function (val, cb) { + var b = Buffer.isBuffer(val) && val.length === this._size; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +FixedType.prototype._read = function (tap) { + return tap.readFixed(this._size); +}; + +FixedType.prototype._skip = function (tap) { + tap.skipFixed(this._size); +}; + +FixedType.prototype._write = function (tap, val) { + if (!Buffer.isBuffer(val) || val.length !== this._size) { + throwInvalidError(null, val, this); + } + tap.writeFixed(val, this._size); +}; + +FixedType.prototype._match = function (tap1, tap2) { + return tap1.matchFixed(tap2, this._size); +}; + +FixedType.prototype.compare = Buffer.compare; + +FixedType.prototype._updateResolver = function (resolver, type) { + if ( + type instanceof FixedType && + this._size === type._size && + ~getAliases(this).indexOf(type._name) + ) { + resolver._size = this._size; + resolver._read = this._read; + } +}; + +FixedType.prototype._copy = BytesType.prototype._copy; + +FixedType.prototype.getAliases = function () { return this._aliases; }; + +FixedType.prototype.getSize = function () { return this._size; }; + +FixedType.prototype.random = function () { + return RANDOM.nextBuffer(this._size); +}; + +FixedType.prototype.toJSON = function () { + return {name: this._name, type: 'fixed', size: this._size}; +}; + +/** + * Avro map. + * + * Represented as vanilla objects. + * + */ +function MapType(attrs, opts) { + if (!attrs.values) { + throw new Error(f('missing map values: %j', attrs)); + } + + opts = getOpts(attrs, opts); + Type.call(this); + this._values = createType(attrs.values, opts); +} +util.inherits(MapType, Type); + +MapType.prototype.getValuesType = function () { return this._values; }; + +MapType.prototype._check = function (val, cb) { + if (!val || typeof val != 'object' || val instanceof Array) { + if (cb) { + cb(PATH.slice(), val, this); + } + return false; + } + + var keys = Object.keys(val); + var b = true; + var i, l, j, key; + if (cb) { + // Slow path. + j = PATH.length; + PATH.push(''); + for (i = 0, l = keys.length; i < l; i++) { + key = PATH[j] = keys[i]; + if (!this._values._check(val[key], cb)) { + b = false; + } + } + PATH.pop(); + } else { + for (i = 0, l = keys.length; i < l; i++) { + if (!this._values._check(val[keys[i]], cb)) { + return false; + } + } + } + return b; +}; + +MapType.prototype._read = function (tap) { + var values = this._values; + var val = {}; + var n; + while ((n = readArraySize(tap))) { + while (n--) { + var key = tap.readString(); + val[key] = values._read(tap); + } + } + return val; +}; + +MapType.prototype._skip = function (tap) { + var values = this._values; + var len, n; + while ((n = tap.readLong())) { + if (n < 0) { + len = tap.readLong(); + tap.pos += len; + } else { + while (n--) { + tap.skipString(); + values._skip(tap); + } + } + } +}; + +MapType.prototype._write = function (tap, val) { + if (!val || typeof val != 'object' || val instanceof Array) { + throwInvalidError(null, val, this); + } + + var values = this._values; + var keys = Object.keys(val); + var n = keys.length; + var i, key; + if (n) { + tap.writeLong(n); + for (i = 0; i < n; i++) { + key = keys[i]; + tap.writeString(key); + values._write(tap, val[key]); + } + } + tap.writeLong(0); +}; + +MapType.prototype._match = function () { + throw new Error('maps cannot be compared'); +}; + +MapType.prototype._updateResolver = function (resolver, type, opts) { + if (type instanceof MapType) { + resolver._values = this._values.createResolver(type._values, opts); + resolver._read = this._read; + } +}; + +MapType.prototype._copy = function (val, opts) { + if (val && typeof val == 'object' && !(val instanceof Array)) { + var values = this._values; + var keys = Object.keys(val); + var i, l, key; + var copy = {}; + for (i = 0, l = keys.length; i < l; i++) { + key = keys[i]; + copy[key] = values._copy(val[key], opts); + } + return copy; + } + throwInvalidError(null, val, this); +}; + +MapType.prototype.compare = MapType.prototype._match; + +MapType.prototype.random = function () { + var val = {}; + var i, l; + for (i = 0, l = RANDOM.nextInt(10); i < l; i++) { + val[RANDOM.nextString(RANDOM.nextInt(20))] = this._values.random(); + } + return val; +}; + +MapType.prototype.toJSON = function () { + return {type: 'map', values: this._values}; +}; + +/** + * Avro array. + * + * Represented as vanilla arrays. + * + */ +function ArrayType(attrs, opts) { + if (!attrs.items) { + throw new Error(f('missing array items: %j', attrs)); + } + + opts = getOpts(attrs, opts); + + this._items = createType(attrs.items, opts); + Type.call(this); +} +util.inherits(ArrayType, Type); + +ArrayType.prototype._check = function (val, cb) { + if (!(val instanceof Array)) { + if (cb) { + cb(PATH.slice(), val, this); + } + return false; + } + + var b = true; + var i, l, j; + if (cb) { + // Slow path. + j = PATH.length; + PATH.push(''); + for (i = 0, l = val.length; i < l; i++) { + PATH[j] = '' + i; + if (!this._items._check(val[i], cb)) { + b = false; + } + } + PATH.pop(); + } else { + for (i = 0, l = val.length; i < l; i++) { + if (!this._items._check(val[i], cb)) { + return false; + } + } + } + return b; +}; + +ArrayType.prototype._read = function (tap) { + var items = this._items; + var val = []; + var n; + while ((n = tap.readLong())) { + if (n < 0) { + n = -n; + tap.skipLong(); // Skip size. + } + while (n--) { + val.push(items._read(tap)); + } + } + return val; +}; + +ArrayType.prototype._skip = function (tap) { + var len, n; + while ((n = tap.readLong())) { + if (n < 0) { + len = tap.readLong(); + tap.pos += len; + } else { + while (n--) { + this._items._skip(tap); + } + } + } +}; + +ArrayType.prototype._write = function (tap, val) { + if (!(val instanceof Array)) { + throwInvalidError(null, val, this); + } + + var n = val.length; + var i; + if (n) { + tap.writeLong(n); + for (i = 0; i < n; i++) { + this._items._write(tap, val[i]); + } + } + tap.writeLong(0); +}; + +ArrayType.prototype._match = function (tap1, tap2) { + var n1 = tap1.readLong(); + var n2 = tap2.readLong(); + var f; + while (n1 && n2) { + f = this._items._match(tap1, tap2); + if (f) { + return f; + } + if (!--n1) { + n1 = readArraySize(tap1); + } + if (!--n2) { + n2 = readArraySize(tap2); + } + } + return utils.compare(n1, n2); +}; + +ArrayType.prototype._updateResolver = function (resolver, type, opts) { + if (type instanceof ArrayType) { + resolver._items = this._items.createResolver(type._items, opts); + resolver._read = this._read; + } +}; + +ArrayType.prototype._copy = function (val, opts) { + if (!(val instanceof Array)) { + throwInvalidError(null, val, this); + } + var items = []; + var i, l; + for (i = 0, l = val.length; i < l; i++) { + items.push(this._items._copy(val[i], opts)); + } + return items; +}; + +ArrayType.prototype.compare = function (val1, val2) { + var n1 = val1.length; + var n2 = val2.length; + var i, l, f; + for (i = 0, l = Math.min(n1, n2); i < l; i++) { + if ((f = this._items.compare(val1[i], val2[i]))) { + return f; + } + } + return utils.compare(n1, n2); +}; + +ArrayType.prototype.getItemsType = function () { return this._items; }; + +ArrayType.prototype.random = function () { + var arr = []; + var i, l; + for (i = 0, l = RANDOM.nextInt(10); i < l; i++) { + arr.push(this._items.random()); + } + return arr; +}; + +ArrayType.prototype.toJSON = function () { + return {type: 'array', items: this._items}; +}; + +/** + * Avro record. + * + * Values are represented as instances of a programmatically generated + * constructor (similar to a "specific record"), available via the + * `getRecordConstructor` method. This "specific record class" gives + * significant speedups over using generics objects. + * + * Note that vanilla objects are still accepted as valid as long as their + * fields match (this makes it much more convenient to do simple things like + * update nested records). + * + */ +function RecordType(attrs, opts) { + opts = getOpts(attrs, opts); + + var resolutions = resolveNames(attrs, opts.namespace); + this._name = resolutions.name; + this._aliases = resolutions.aliases; + Type.call(this, opts.registry); + + if (!(attrs.fields instanceof Array)) { + throw new Error(f('non-array %s fields', this._name)); + } + this._fields = attrs.fields.map(function (f) { + return new Field(f, opts); + }); + if (utils.hasDuplicates(attrs.fields, function (f) { return f.name; })) { + throw new Error(f('duplicate %s field name', this._name)); + } + + var isError = attrs.type === 'error'; + this._constructor = this._createConstructor(isError); + this._read = this._createReader(); + this._skip = this._createSkipper(); + this._write = this._createWriter(); + this._check = this._createChecker(); +} +util.inherits(RecordType, Type); + +RecordType.prototype._createConstructor = function (isError) { + // jshint -W054 + var outerArgs = []; + var innerArgs = []; + var ds = []; // Defaults. + var innerBody = isError ? ' Error.call(this);\n' : ''; + // Not calling `Error.captureStackTrace` because this wouldn't be compatible + // with browsers other than Chrome. + var i, l, field, name, getDefault; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + getDefault = field.getDefault; + name = field._name; + innerArgs.push('v' + i); + innerBody += ' '; + if (getDefault() === undefined) { + innerBody += 'this.' + name + ' = v' + i + ';\n'; + } else { + innerBody += 'if (v' + i + ' === undefined) { '; + innerBody += 'this.' + name + ' = d' + ds.length + '(); '; + innerBody += '} else { this.' + name + ' = v' + i + '; }\n'; + outerArgs.push('d' + ds.length); + ds.push(getDefault); + } + } + var outerBody = 'return function ' + unqualify(this._name) + '('; + outerBody += innerArgs.join() + ') {\n' + innerBody + '};'; + var Record = new Function(outerArgs.join(), outerBody).apply(undefined, ds); + + var self = this; + Record.getType = function () { return self; }; + Record.prototype = { + constructor: Record, + $clone: function (opts) { return self.clone(this, opts); }, + $compare: function (val) { return self.compare(this, val); }, + $getType: Record.getType, + $isValid: function (opts) { return self.isValid(this, opts); }, + $toBuffer: function () { return self.toBuffer(this); }, + $toString: function (noCheck) { return self.toString(this, noCheck); } + }; + // The names of these properties added to the prototype are prefixed with `$` + // because it is an invalid property name in Avro but not in JavaScript. + // (This way we are guaranteed not to be stepped over!) + if (isError) { + util.inherits(Record, Error); + // Not setting the name on the prototype to be consistent with how object + // fields are mapped to (only if defined in the schema as a field). + } + + return Record; +}; + +RecordType.prototype._createChecker = function () { + // jshint -W054 + var names = ['t', 'P']; + var values = [this, PATH]; + var body = 'return function check' + unqualify(this._name) + '(val, cb) {\n'; + body += ' if (val === null || typeof val != \'object\') {\n'; + body += ' if (cb) { cb(P.slice(), val, t); }\n'; + body += ' return false;\n'; + body += ' }\n'; + if (!this._fields.length) { + // Special case, empty record. We handle this directly. + body += ' return true;\n'; + } else { + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + names.push('t' + i); + values.push(field._type); + if (field.getDefault() !== undefined) { + body += ' var v' + i + ' = val.' + field._name + ';\n'; + } + } + body += ' if (cb) {\n'; + body += ' var b = 1;\n'; + body += ' var j = P.length;\n'; + body += ' P.push(\'\');\n'; + var i, l, field; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + body += ' P[j] = \'' + field._name + '\';\n'; + if (field.getDefault() === undefined) { + body += ' b &= t' + i + '._check(val.' + field._name + ', cb);\n'; + } else { + body += ' b &= v' + i + ' === undefined || '; + body += 't' + i + '._check(v' + i + ', cb);\n'; + } + } + body += ' P.pop();\n'; + body += ' return !!b;\n'; + body += ' } else {\n return (\n '; + body += this._fields.map(function (field, i) { + if (field.getDefault() === undefined) { + return 't' + i + '._check(val.' + field._name + ')'; + } else { + return '(v' + i + ' === undefined || t' + i + '._check(v' + i + '))'; + } + }).join(' &&\n '); + body += '\n );\n }\n'; + } + body += '};'; + return new Function(names.join(), body).apply(undefined, values); +}; + +RecordType.prototype._createReader = function () { + // jshint -W054 + var uname = unqualify(this._name); + var names = []; + var values = [this._constructor]; + var i, l; + for (i = 0, l = this._fields.length; i < l; i++) { + names.push('t' + i); + values.push(this._fields[i]._type); + } + var body = 'return function read' + uname + '(tap) {\n'; + body += ' return new ' + uname + '('; + body += names.map(function (t) { return t + '._read(tap)'; }).join(); + body += ');\n};'; + names.unshift(uname); + // We can do this since the JS spec guarantees that function arguments are + // evaluated from left to right. + return new Function(names.join(), body).apply(undefined, values); +}; + +RecordType.prototype._createSkipper = function () { + // jshint -W054 + var args = []; + var body = 'return function skip' + unqualify(this._name) + '(tap) {\n'; + var values = []; + var i, l; + for (i = 0, l = this._fields.length; i < l; i++) { + args.push('t' + i); + values.push(this._fields[i]._type); + body += ' t' + i + '._skip(tap);\n'; + } + body += '}'; + return new Function(args.join(), body).apply(undefined, values); +}; + +RecordType.prototype._createWriter = function () { + // jshint -W054 + // We still do default handling here, in case a normal JS object is passed. + var args = []; + var body = 'return function write' + unqualify(this._name) + '(tap, val) {\n'; + var values = []; + var i, l, field, value; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + args.push('t' + i); + values.push(field._type); + body += ' '; + if (field.getDefault() === undefined) { + body += 't' + i + '._write(tap, val.' + field._name + ');\n'; + } else { + value = field._type.toBuffer(field.getDefault()).toString('binary'); + // Convert the default value to a binary string ahead of time. We aren't + // converting it to a buffer to avoid retaining too much memory. If we + // had our own buffer pool, this could be an idea in the future. + args.push('d' + i); + values.push(value); + body += 'var v' + i + ' = val.' + field._name + '; '; + body += 'if (v' + i + ' === undefined) { '; + body += 'tap.writeBinary(d' + i + ', ' + value.length + ');'; + body += ' } else { t' + i + '._write(tap, v' + i + '); }\n'; + } + } + body += '}'; + return new Function(args.join(), body).apply(undefined, values); +}; + +RecordType.prototype._updateResolver = function (resolver, type, opts) { + // jshint -W054 + if (!~getAliases(this).indexOf(type._name)) { + throw new Error(f('no alias for %s in %s', type._name, this._name)); + } + + var rFields = this._fields; + var wFields = type._fields; + var wFieldsMap = utils.toMap(wFields, function (f) { return f._name; }); + + var innerArgs = []; // Arguments for reader constructor. + var resolvers = {}; // Resolvers keyed by writer field name. + var i, j, field, name, names, matches; + for (i = 0; i < rFields.length; i++) { + field = rFields[i]; + names = getAliases(field); + matches = []; + for (j = 0; j < names.length; j++) { + name = names[j]; + if (wFieldsMap[name]) { + matches.push(name); + } + } + if (matches.length > 1) { + throw new Error(f('multiple matches for %s', field.name)); + } + if (!matches.length) { + if (field.getDefault() === undefined) { + throw new Error(f('no match for default-less %s', field.name)); + } + innerArgs.push('undefined'); + } else { + name = matches[0]; + resolvers[name] = { + resolver: field._type.createResolver(wFieldsMap[name]._type, opts), + name: field._name // Reader field name. + }; + innerArgs.push(field._name); + } + } + + // See if we can add a bypass for unused fields at the end of the record. + var lazyIndex = -1; + i = wFields.length; + while (i && resolvers[wFields[--i]._name] === undefined) { + lazyIndex = i; + } + + var uname = unqualify(this._name); + var args = [uname]; + var values = [this._constructor]; + var body = ' return function read' + uname + '(tap,lazy) {\n'; + for (i = 0; i < wFields.length; i++) { + if (i === lazyIndex) { + body += ' if (!lazy) {\n'; + } + field = type._fields[i]; + name = field._name; + body += (~lazyIndex && i >= lazyIndex) ? ' ' : ' '; + if (resolvers[name] === undefined) { + args.push('t' + i); + values.push(field._type); + body += 't' + i + '._skip(tap);\n'; + } else { + args.push('t' + i); + values.push(resolvers[name].resolver); + body += 'var ' + resolvers[name].name + ' = '; + body += 't' + i + '._read(tap);\n'; + } + } + if (~lazyIndex) { + body += ' }\n'; + } + body += ' return new ' + uname + '(' + innerArgs.join() + ');\n};'; + + resolver._read = new Function(args.join(), body).apply(undefined, values); +}; + +RecordType.prototype._match = function (tap1, tap2) { + var fields = this._fields; + var i, l, field, order, type; + for (i = 0, l = fields.length; i < l; i++) { + field = fields[i]; + order = field._order; + type = field._type; + if (order) { + order *= type._match(tap1, tap2); + if (order) { + return order; + } + } else { + type._skip(tap1); + type._skip(tap2); + } + } + return 0; +}; + +RecordType.prototype._copy = function (val, opts) { + // jshint -W058 + var hook = opts && opts.fieldHook; + var values = [undefined]; + var i, l, field, value; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + value = field._type._copy(val[field._name], opts); + if (hook) { + value = hook(field, value, this); + } + values.push(value); + } + return new (this._constructor.bind.apply(this._constructor, values)); +}; + +RecordType.prototype.compare = function (val1, val2) { + var fields = this._fields; + var i, l, field, name, order, type; + for (i = 0, l = fields.length; i < l; i++) { + field = fields[i]; + name = field._name; + order = field._order; + type = field._type; + if (order) { + order *= type.compare(val1[name], val2[name]); + if (order) { + return order; + } + } + } + return 0; +}; + +RecordType.prototype.random = function () { + // jshint -W058 + var fields = this._fields.map(function (f) { return f._type.random(); }); + fields.unshift(undefined); + return new (this._constructor.bind.apply(this._constructor, fields)); +}; + +RecordType.prototype.getAliases = function () { return this._aliases; }; + +RecordType.prototype.getFields = function () { return this._fields.slice(); }; + +RecordType.prototype.getRecordConstructor = function () { + return this._constructor; +}; + +RecordType.prototype.toJSON = function () { + return {name: this._name, type: 'record', fields: this._fields}; +}; + +/** + * Derived type abstract class. + * + */ +function LogicalType(attrs, opts, Types) { + Type.call(this); + LOGICAL_TYPE = this; + this._underlyingType = createType(attrs, opts); + + // Convenience type check. + if (Types && !~Types.indexOf(this._underlyingType.constructor)) { + var lType = attrs.logicalType; + var uType = this._underlyingType; + throw new Error(f('invalid underlying type for %s: %s', lType, uType)); + } +} +util.inherits(LogicalType, Type); + +LogicalType.prototype.getUnderlyingType = function () { + return this._underlyingType; +}; + +LogicalType.prototype._read = function (tap) { + return this._fromValue(this._underlyingType._read(tap)); +}; + +LogicalType.prototype._write = function (tap, val) { + this._underlyingType._write(tap, this._toValue(val)); +}; + +LogicalType.prototype._check = function (val, cb) { + return this._underlyingType._check(this._toValue(val), cb); +}; + +LogicalType.prototype._copy = function (val, opts) { + var type = this._underlyingType; + switch (opts && opts.coerce) { + case 3: // To string. + return type._copy(this._toValue(val), opts); + case 2: // From string. + return this._fromValue(type._copy(val, opts)); + default: // Normal copy. + return this._fromValue(type._copy(this._toValue(val), opts)); + } +}; + +LogicalType.prototype._updateResolver = function (resolver, type, opts) { + var _fromValue = this._resolve(type, opts); + if (_fromValue) { + resolver._read = function (tap) { return _fromValue(type._read(tap)); }; + } +}; + +LogicalType.prototype.random = function () { + return this._fromValue(this._underlyingType.random()); +}; + +LogicalType.prototype.compare = function (obj1, obj2) { + var val1 = this._toValue(obj1); + var val2 = this._toValue(obj2); + return this._underlyingType.compare(val1, val2); +}; + +LogicalType.prototype.toJSON = function () { + return this._underlyingType.toJSON(); +}; + +// Methods to be implemented. +LogicalType.prototype._fromValue = utils.abstractFunction; +LogicalType.prototype._toValue = utils.abstractFunction; +LogicalType.prototype._resolve = utils.abstractFunction; + + +// General helpers. + +/** + * Customizable long. + * + * This allows support of arbitrarily large long (e.g. larger than + * `Number.MAX_SAFE_INTEGER`). See `LongType.using` method above. + * + */ +function AbstractLongType(noUnpack) { + LongType.call(this); + this._noUnpack = !!noUnpack; +} +util.inherits(AbstractLongType, LongType); + +AbstractLongType.prototype._check = function (val, cb) { + var b = this._isValid(val); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +AbstractLongType.prototype._read = function (tap) { + var buf, pos; + if (this._noUnpack) { + pos = tap.pos; + tap.skipLong(); + buf = tap.buf.slice(pos, tap.pos); + } else { + buf = tap.unpackLongBytes(tap); + } + if (tap.isValid()) { + return this._fromBuffer(buf); + } +}; + +AbstractLongType.prototype._write = function (tap, val) { + if (!this._isValid(val)) { + throwInvalidError(null, val, this); + } + var buf = this._toBuffer(val); + if (this._noUnpack) { + tap.writeFixed(buf); + } else { + tap.packLongBytes(buf); + } +}; + +AbstractLongType.prototype._copy = function (val, opts) { + switch (opts && opts.coerce) { + case 3: // To string. + return this._toJSON(val); + case 2: // From string. + return this._fromJSON(val); + default: // Normal copy. + // Slow but guarantees most consistent results. Faster alternatives would + // require assumptions on the long class used (e.g. immutability). + return this._fromJSON(JSON.parse(JSON.stringify(this._toJSON(val)))); + } +}; + +AbstractLongType.prototype.random = function () { + return this._fromJSON(LongType.prototype.random()); +}; + +// Methods to be implemented by the user. +AbstractLongType.prototype._fromBuffer = utils.abstractFunction; +AbstractLongType.prototype._toBuffer = utils.abstractFunction; +AbstractLongType.prototype._fromJSON = utils.abstractFunction; +AbstractLongType.prototype._toJSON = utils.abstractFunction; +AbstractLongType.prototype._isValid = utils.abstractFunction; +AbstractLongType.prototype.compare = utils.abstractFunction; + +/** + * Field. + * + * @param attrs {Object} The field's schema. + * @para opts {Object} Schema parsing options (the same as `Type`s'). + * + */ +function Field(attrs, opts) { + var name = attrs.name; + if (typeof name != 'string' || !NAME_PATTERN.test(name)) { + throw new Error(f('invalid field name: %s', name)); + } + + this._name = name; + this._type = createType(attrs.type, opts); + this._aliases = attrs.aliases || []; + + this._order = (function (order) { + switch (order) { + case 'ascending': + return 1; + case 'descending': + return -1; + case 'ignore': + return 0; + default: + throw new Error(f('invalid order: %j', order)); + } + })(attrs.order === undefined ? 'ascending' : attrs.order); + + var value = attrs['default']; + if (value !== undefined) { + // We need to convert defaults back to a valid format (unions are + // disallowed in default definitions, only the first type of each union is + // allowed instead). + // http://apache-avro.679487.n3.nabble.com/field-union-default-in-Java-td1175327.html + var type = this._type; + var val = type._copy(value, {coerce: 2, wrap: 2}); + // The clone call above will throw an error if the default is invalid. + if (type instanceof PrimitiveType && !(type instanceof BytesType)) { + // These are immutable. + this.getDefault = function () { return val; }; + } else { + this.getDefault = function () { return type._copy(val); }; + } + } +} + +Field.prototype.getAliases = function () { return this._aliases; }; + +Field.prototype.getDefault = function () {}; // Undefined default. + +Field.prototype.getName = function () { return this._name; }; + +Field.prototype.getOrder = function () { + return ['descending', 'ignore', 'ascending'][this._order + 1]; +}; + +Field.prototype.getType = function () { return this._type; }; + +Field.prototype.inspect = function () { return f('', this._name); }; + +/** + * Resolver to read a writer's schema as a new schema. + * + * @param readerType {Type} The type to convert to. + * + */ +function Resolver(readerType) { + // Add all fields here so that all resolvers share the same hidden class. + this._readerType = readerType; + this._items = null; + this._read = null; + this._size = 0; + this._symbols = null; + this._values = null; +} + +Resolver.prototype.inspect = function () { return ''; }; + +/** + * Read a value from a tap. + * + * @param type {Type} The type to decode. + * @param tap {Tap} The tap to read from. No checks are performed here. + * @param resolver {Resolver} Optional resolver. It must match the input type. + * @param lazy {Boolean} Skip trailing fields when using a resolver. + * + */ +function readValue(type, tap, resolver, lazy) { + if (resolver) { + if (resolver._readerType !== type) { + throw new Error('invalid resolver'); + } + return resolver._read(tap, lazy); + } else { + return type._read(tap); + } +} + +/** + * Create default parsing options. + * + * @param attrs {Object} Schema to populate options with. + * @param opts {Object} Base options. + * + */ +function getOpts(attrs, opts) { + if (attrs === null) { + // Let's be helpful for this common error. + throw new Error('invalid type: null (did you mean "null"?)'); + } + opts = opts || {}; + opts.registry = opts.registry || {}; + opts.namespace = attrs.namespace || opts.namespace; + opts.logicalTypes = opts.logicalTypes || {}; + return opts; +} + +/** + * Resolve a schema's name and aliases. + * + * @param attrs {Object} True schema (can't be a string). + * @param namespace {String} Optional parent namespace. + * @param key {String} Key where the name should be looked up (defaults to + * `name`). + * + */ +function resolveNames(attrs, namespace, key) { + namespace = attrs.namespace || namespace; + key = key || 'name'; + + var name = attrs[key]; + if (!name) { + throw new Error(f('missing %s property in schema: %j', key, attrs)); + } + return { + name: qualify(name), + aliases: attrs.aliases ? attrs.aliases.map(qualify) : [] + }; + + function qualify(name) { + if (!~name.indexOf('.') && namespace) { + name = namespace + '.' + name; + } + var tail = unqualify(name); + if (isPrimitive(tail)) { + // Primitive types cannot be defined in any namespace. + throw new Error(f('cannot rename primitive type: %j', tail)); + } + name.split('.').forEach(function (part) { + if (!NAME_PATTERN.test(part)) { + throw new Error(f('invalid name: %j', name)); + } + }); + return name; + } +} + +/** + * Remove namespace from a name. + * + * @param name {String} Full or short name. + * + */ +function unqualify(name) { + var parts = name.split('.'); + return parts[parts.length - 1]; +} + +/** + * Get all aliases for a type (including its name). + * + * @param obj {Type|Object} Typically a type or a field. Its aliases property + * must exist and be an array. + * + */ +function getAliases(obj) { + var names = [obj._name]; + var aliases = obj._aliases; + var i, l; + for (i = 0, l = aliases.length; i < l; i++) { + names.push(aliases[i]); + } + return names; +} + +/** + * Get a type's "type" (as a string, e.g. `'record'`, `'string'`). + * + * @param type {Type} Any type. + * + */ +function getTypeName(type) { + var obj = type.toJSON(); + return typeof obj == 'string' ? obj : obj.type; +} + +/** + * Check whether a type's name is a primitive. + * + * @param name {String} Type name (e.g. `'string'`, `'array'`). + * + */ +function isPrimitive(name) { + var type = TYPES[name]; + return type !== undefined && type.prototype instanceof PrimitiveType; +} + +/** + * Get the number of elements in an array block. + * + * @param tap {Tap} A tap positioned at the beginning of an array block. + * + */ +function readArraySize(tap) { + var n = tap.readLong(); + if (n < 0) { + n = -n; + tap.skipLong(); // Skip size. + } + return n; +} + +/** + * Check whether a long can be represented without precision loss. + * + * @param n {Number} The number. + * + * Two things to note: + * + * + We are not using the `Number` constants for compatibility with older + * browsers. + * + We must remove one from each bound because of rounding errors. + * + */ +function isSafeLong(n) { + return n >= -9007199254740990 && n <= 9007199254740990; +} + +/** + * Throw a somewhat helpful error on invalid object. + * + * @param path {Array} Passed from hook, but unused (because empty where this + * function is used, since we aren't keeping track of it for effiency). + * @param val {...} The object to reject. + * @param type {Type} The type to check against. + * + * This method is mostly used from `_write` to signal an invalid object for a + * given type. Note that this provides less information than calling `isValid` + * with a hook since the path is not propagated (for efficiency reasons). + * + */ +function throwInvalidError(path, val, type) { + throw new Error(f('invalid %s: %j', type, val)); +} + + +module.exports = { + createType: createType, + resolveNames: resolveNames, // Protocols use the same name resolution logic. + types: (function () { + // Export the base types along with all concrete implementations. + var obj = {Type: Type, LogicalType: LogicalType}; + var types = Object.keys(TYPES); + var i, l, Class; + for (i = 0, l = types.length; i < l; i++) { + Class = TYPES[types[i]]; + obj[Class.name] = Class; + } + return obj; + })() +}; diff --git a/lang/js/lib/utils.js b/lang/js/lib/utils.js new file mode 100644 index 00000000000..2627c1aa5f1 --- /dev/null +++ b/lang/js/lib/utils.js @@ -0,0 +1,632 @@ +/* jshint node: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +/** + * Uppercase the first letter of a string. + * + * @param s {String} The string. + * + */ +function capitalize(s) { return s.charAt(0).toUpperCase() + s.slice(1); } + +/** + * Compare two numbers. + * + * @param n1 {Number} The first one. + * @param n2 {Number} The second one. + * + */ +function compare(n1, n2) { return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); } + +/** + * Find index of value in array. + * + * @param arr {Array} Can also be a false-ish value. + * @param v {Object} Value to find. + * + * Returns -1 if not found, -2 if found multiple times. + * + */ +function singleIndexOf(arr, v) { + var pos = -1; + var i, l; + if (!arr) { + return -1; + } + for (i = 0, l = arr.length; i < l; i++) { + if (arr[i] === v) { + if (pos >= 0) { + return -2; + } + pos = i; + } + } + return pos; +} + +/** + * Convert array to map. + * + * @param arr {Array} Elements. + * @param fn {Function} Function returning an element's key. + * + */ +function toMap(arr, fn) { + var obj = {}; + var i, elem; + for (i = 0; i < arr.length; i++) { + elem = arr[i]; + obj[fn(elem)] = elem; + } + return obj; +} + +/** + * Check whether an array has duplicates. + * + * @param arr {Array} The array. + * @param fn {Function} Optional function to apply to each element. + * + */ +function hasDuplicates(arr, fn) { + var obj = {}; + var i, l, elem; + for (i = 0, l = arr.length; i < l; i++) { + elem = arr[i]; + if (fn) { + elem = fn(elem); + } + if (obj[elem]) { + return true; + } + obj[elem] = true; + } + return false; +} + +/** + * "Abstract" function to help with "subclassing". + * + */ +function abstractFunction() { throw new Error('abstract'); } + +/** + * Generator of random things. + * + * Inspired by: http://stackoverflow.com/a/424445/1062617 + * + */ +function Lcg(seed) { + var a = 1103515245; + var c = 12345; + var m = Math.pow(2, 31); + var state = Math.floor(seed || Math.random() * (m - 1)); + + this._max = m; + this._nextInt = function () { return state = (a * state + c) % m; }; +} + +Lcg.prototype.nextBoolean = function () { + // jshint -W018 + return !!(this._nextInt() % 2); +}; + +Lcg.prototype.nextInt = function (start, end) { + if (end === undefined) { + end = start; + start = 0; + } + end = end === undefined ? this._max : end; + return start + Math.floor(this.nextFloat() * (end - start)); +}; + +Lcg.prototype.nextFloat = function (start, end) { + if (end === undefined) { + end = start; + start = 0; + } + end = end === undefined ? 1 : end; + return start + (end - start) * this._nextInt() / this._max; +}; + +Lcg.prototype.nextString = function(len, flags) { + len |= 0; + flags = flags || 'aA'; + var mask = ''; + if (flags.indexOf('a') > -1) { + mask += 'abcdefghijklmnopqrstuvwxyz'; + } + if (flags.indexOf('A') > -1) { + mask += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; + } + if (flags.indexOf('#') > -1) { + mask += '0123456789'; + } + if (flags.indexOf('!') > -1) { + mask += '~`!@#$%^&*()_+-={}[]:";\'<>?,./|\\'; + } + var result = []; + for (var i = 0; i < len; i++) { + result.push(this.choice(mask)); + } + return result.join(''); +}; + +Lcg.prototype.nextBuffer = function (len) { + var arr = []; + var i; + for (i = 0; i < len; i++) { + arr.push(this.nextInt(256)); + } + return new Buffer(arr); +}; + +Lcg.prototype.choice = function (arr) { + var len = arr.length; + if (!len) { + throw new Error('choosing from empty array'); + } + return arr[this.nextInt(len)]; +}; + +/** + * Ordered queue which returns items consecutively. + * + * This is actually a heap by index, with the added requirements that elements + * can only be retrieved consecutively. + * + */ +function OrderedQueue() { + this._index = 0; + this._items = []; +} + +OrderedQueue.prototype.push = function (item) { + var items = this._items; + var i = items.length | 0; + var j; + items.push(item); + while (i > 0 && items[i].index < items[j = ((i - 1) >> 1)].index) { + item = items[i]; + items[i] = items[j]; + items[j] = item; + i = j; + } +}; + +OrderedQueue.prototype.pop = function () { + var items = this._items; + var len = (items.length - 1) | 0; + var first = items[0]; + if (!first || first.index > this._index) { + return null; + } + this._index++; + if (!len) { + items.pop(); + return first; + } + items[0] = items.pop(); + var mid = len >> 1; + var i = 0; + var i1, i2, j, item, c, c1, c2; + while (i < mid) { + item = items[i]; + i1 = (i << 1) + 1; + i2 = (i + 1) << 1; + c1 = items[i1]; + c2 = items[i2]; + if (!c2 || c1.index <= c2.index) { + c = c1; + j = i1; + } else { + c = c2; + j = i2; + } + if (c.index >= item.index) { + break; + } + items[j] = item; + items[i] = c; + i = j; + } + return first; +}; + +/** + * A tap is a buffer which remembers what has been already read. + * + * It is optimized for performance, at the cost of failing silently when + * overflowing the buffer. This is a purposeful trade-off given the expected + * rarity of this case and the large performance hit necessary to enforce + * validity. See `isValid` below for more information. + * + */ +function Tap(buf, pos) { + this.buf = buf; + this.pos = pos | 0; +} + +/** + * Check that the tap is in a valid state. + * + * For efficiency reasons, none of the methods below will fail if an overflow + * occurs (either read, skip, or write). For this reason, it is up to the + * caller to always check that the read, skip, or write was valid by calling + * this method. + * + */ +Tap.prototype.isValid = function () { return this.pos <= this.buf.length; }; + +// Read, skip, write methods. +// +// These should fail silently when the buffer overflows. Note this is only +// required to be true when the functions are decoding valid objects. For +// example errors will still be thrown if a bad count is read, leading to a +// negative position offset (which will typically cause a failure in +// `readFixed`). + +Tap.prototype.readBoolean = function () { return !!this.buf[this.pos++]; }; + +Tap.prototype.skipBoolean = function () { this.pos++; }; + +Tap.prototype.writeBoolean = function (b) { this.buf[this.pos++] = !!b; }; + +Tap.prototype.readInt = Tap.prototype.readLong = function () { + var n = 0; + var k = 0; + var buf = this.buf; + var b, h, f, fk; + + do { + b = buf[this.pos++]; + h = b & 0x80; + n |= (b & 0x7f) << k; + k += 7; + } while (h && k < 28); + + if (h) { + // Switch to float arithmetic, otherwise we might overflow. + f = n; + fk = 268435456; // 2 ** 28. + do { + b = buf[this.pos++]; + f += (b & 0x7f) * fk; + fk *= 128; + } while (b & 0x80); + return (f % 2 ? -(f + 1) : f) / 2; + } + + return (n >> 1) ^ -(n & 1); +}; + +Tap.prototype.skipInt = Tap.prototype.skipLong = function () { + var buf = this.buf; + while (buf[this.pos++] & 0x80) {} +}; + +Tap.prototype.writeInt = Tap.prototype.writeLong = function (n) { + var buf = this.buf; + var f, m; + + if (n >= -1073741824 && n < 1073741824) { + // Won't overflow, we can use integer arithmetic. + m = n >= 0 ? n << 1 : (~n << 1) | 1; + do { + buf[this.pos] = m & 0x7f; + m >>= 7; + } while (m && (buf[this.pos++] |= 0x80)); + } else { + // We have to use slower floating arithmetic. + f = n >= 0 ? n * 2 : (-n * 2) - 1; + do { + buf[this.pos] = f & 0x7f; + f /= 128; + } while (f >= 1 && (buf[this.pos++] |= 0x80)); + } + this.pos++; +}; + +Tap.prototype.readFloat = function () { + var buf = this.buf; + var pos = this.pos; + this.pos += 4; + if (this.pos > buf.length) { + return; + } + return this.buf.readFloatLE(pos); +}; + +Tap.prototype.skipFloat = function () { this.pos += 4; }; + +Tap.prototype.writeFloat = function (f) { + var buf = this.buf; + var pos = this.pos; + this.pos += 4; + if (this.pos > buf.length) { + return; + } + return this.buf.writeFloatLE(f, pos); +}; + +Tap.prototype.readDouble = function () { + var buf = this.buf; + var pos = this.pos; + this.pos += 8; + if (this.pos > buf.length) { + return; + } + return this.buf.readDoubleLE(pos); +}; + +Tap.prototype.skipDouble = function () { this.pos += 8; }; + +Tap.prototype.writeDouble = function (d) { + var buf = this.buf; + var pos = this.pos; + this.pos += 8; + if (this.pos > buf.length) { + return; + } + return this.buf.writeDoubleLE(d, pos); +}; + +Tap.prototype.readFixed = function (len) { + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + var fixed = new Buffer(len); + this.buf.copy(fixed, 0, pos, pos + len); + return fixed; +}; + +Tap.prototype.skipFixed = function (len) { this.pos += len; }; + +Tap.prototype.writeFixed = function (buf, len) { + len = len || buf.length; + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + buf.copy(this.buf, pos, 0, len); +}; + +Tap.prototype.readBytes = function () { + return this.readFixed(this.readLong()); +}; + +Tap.prototype.skipBytes = function () { + var len = this.readLong(); + this.pos += len; +}; + +Tap.prototype.writeBytes = function (buf) { + var len = buf.length; + this.writeLong(len); + this.writeFixed(buf, len); +}; + +Tap.prototype.readString = function () { + var len = this.readLong(); + var pos = this.pos; + var buf = this.buf; + this.pos += len; + if (this.pos > buf.length) { + return; + } + return this.buf.utf8Slice(pos, pos + len); +}; + +Tap.prototype.skipString = function () { + var len = this.readLong(); + this.pos += len; +}; + +Tap.prototype.writeString = function (s) { + var len = Buffer.byteLength(s); + this.writeLong(len); + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + this.buf.utf8Write(s, pos, len); +}; + +// Helper used to speed up writing defaults. + +Tap.prototype.writeBinary = function (str, len) { + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + this.buf.binaryWrite(str, pos, len); +}; + +// Binary comparison methods. +// +// These are not guaranteed to consume the objects they are comparing when +// returning a non-zero result (allowing for performance benefits), so no other +// operations should be done on either tap after a compare returns a non-zero +// value. Also, these methods do not have the same silent failure requirement +// as read, skip, and write since they are assumed to be called on valid +// buffers. + +Tap.prototype.matchBoolean = function (tap) { + return this.buf[this.pos++] - tap.buf[tap.pos++]; +}; + +Tap.prototype.matchInt = Tap.prototype.matchLong = function (tap) { + var n1 = this.readLong(); + var n2 = tap.readLong(); + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); +}; + +Tap.prototype.matchFloat = function (tap) { + var n1 = this.readFloat(); + var n2 = tap.readFloat(); + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); +}; + +Tap.prototype.matchDouble = function (tap) { + var n1 = this.readDouble(); + var n2 = tap.readDouble(); + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); +}; + +Tap.prototype.matchFixed = function (tap, len) { + return this.readFixed(len).compare(tap.readFixed(len)); +}; + +Tap.prototype.matchBytes = Tap.prototype.matchString = function (tap) { + var l1 = this.readLong(); + var p1 = this.pos; + this.pos += l1; + var l2 = tap.readLong(); + var p2 = tap.pos; + tap.pos += l2; + var b1 = this.buf.slice(p1, this.pos); + var b2 = tap.buf.slice(p2, tap.pos); + return b1.compare(b2); +}; + +// Functions for supporting custom long classes. +// +// The two following methods allow the long implementations to not have to +// worry about Avro's zigzag encoding, we directly expose longs as unpacked. + +Tap.prototype.unpackLongBytes = function () { + var res = new Buffer(8); + var n = 0; + var i = 0; // Byte index in target buffer. + var j = 6; // Bit offset in current target buffer byte. + var buf = this.buf; + var b, neg; + + b = buf[this.pos++]; + neg = b & 1; + res.fill(0); + + n |= (b & 0x7f) >> 1; + while (b & 0x80) { + b = buf[this.pos++]; + n |= (b & 0x7f) << j; + j += 7; + if (j >= 8) { + // Flush byte. + j -= 8; + res[i++] = n; + n >>= 8; + } + } + res[i] = n; + + if (neg) { + invert(res, 8); + } + + return res; +}; + +Tap.prototype.packLongBytes = function (buf) { + var neg = (buf[7] & 0x80) >> 7; + var res = this.buf; + var j = 1; + var k = 0; + var m = 3; + var n; + + if (neg) { + invert(buf, 8); + n = 1; + } else { + n = 0; + } + + var parts = [ + buf.readUIntLE(0, 3), + buf.readUIntLE(3, 3), + buf.readUIntLE(6, 2) + ]; + // Not reading more than 24 bits because we need to be able to combine the + // "carry" bits from the previous part and JavaScript only supports bitwise + // operations on 32 bit integers. + while (m && !parts[--m]) {} // Skip trailing 0s. + + // Leading parts (if any), we never bail early here since we need the + // continuation bit to be set. + while (k < m) { + n |= parts[k++] << j; + j += 24; + while (j > 7) { + res[this.pos++] = (n & 0x7f) | 0x80; + n >>= 7; + j -= 7; + } + } + + // Final part, similar to normal packing aside from the initial offset. + n |= parts[m] << j; + do { + res[this.pos] = n & 0x7f; + n >>= 7; + } while (n && (res[this.pos++] |= 0x80)); + this.pos++; + + // Restore original buffer (could make this optional?). + if (neg) { + invert(buf, 8); + } +}; + +// Helpers. + +/** + * Invert all bits in a buffer. + * + * @param buf {Buffer} Non-empty buffer to invert. + * @param len {Number} Buffer length (must be positive). + * + */ +function invert(buf, len) { + while (len--) { + buf[len] = ~buf[len]; + } +} + + +module.exports = { + abstractFunction: abstractFunction, + capitalize: capitalize, + compare: compare, + toMap: toMap, + singleIndexOf: singleIndexOf, + hasDuplicates: hasDuplicates, + Lcg: Lcg, + OrderedQueue: OrderedQueue, + Tap: Tap +}; diff --git a/lang/js/package.json b/lang/js/package.json index 39430b1f689..fda5325d6de 100644 --- a/lang/js/package.json +++ b/lang/js/package.json @@ -1,17 +1,18 @@ { "name": "avro-js", - "version": "0.0.1", + "version": "1.8.0", "author": "Avro Developers ", - "description": "Avro validator for Javascript", + "description": "JavaScript Avro implementation", "contributors": [ + { + "name": "Matthieu Monsch", + "email": "monsch@alum.mit.edu" + }, { "name": "Quinn Slack", "email": "sqs@cs.stanford.edu" - } + } ], - "scripts": { - "test": "grunt test" - }, "repository": { "type": "svn", "url": "http://svn.apache.org/repos/asf/avro/trunk/lang/js/" @@ -20,18 +21,34 @@ "avro", "json" ], - "dependencies" : { - "underscore" : "*" + "files": [ + "LICENSE", + "NOTICE", + "lib", + "etc/browser", + "etc/deprecated/validator.js" + ], + "main": "./lib", + "browser": { + "./lib": "./etc/browser/avro.js", + "crypto": "./etc/browser/crypto.js" + }, + "scripts": { + "test": "mocha --ui tdd --reporter dot", + "clean": "rm -rf node_modules" + }, + "dependencies": { + "underscore": "*" }, - "devDependencies" : { - "grunt" : "*", - "grunt-contrib-jshint" : "*", - "grunt-contrib-nodeunit" : "*", - "grunt-contrib-watch" : "*" + "devDependencies": { + "coveralls": "^2.11.4", + "istanbul": "^0.3.19", + "mocha": "^2.3.2", + "tmp": "^0.0.28" }, "noAnalyze": true, - "license": "Apache", + "license": "Apache-2.0", "engine": { - "node": ">=0.4" + "node": ">=0.11" } } diff --git a/lang/js/test/dat/Id.avsc b/lang/js/test/dat/Id.avsc new file mode 100644 index 00000000000..785d4ea1f05 --- /dev/null +++ b/lang/js/test/dat/Id.avsc @@ -0,0 +1,6 @@ +{ + "type": "fixed", + "name": "Id", + "namespace": "id", + "size": 64 +} diff --git a/lang/js/test/dat/Person.avsc b/lang/js/test/dat/Person.avsc new file mode 100644 index 00000000000..2cc8ed3242a --- /dev/null +++ b/lang/js/test/dat/Person.avsc @@ -0,0 +1,20 @@ +{ + "name": "Person", + "type": "record", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": ["null", "int"], "default": null}, + { + "name": "gender", + "type": {"name": "Gender", "type": "enum", "symbols": ["FEMALE", "MALE"]} + }, + { + "name": "address", + "type": { + "name": "Address", + "type": "record", + "fields": [{"name": "zipcode", "type": "int"}] + } + } + ] +} diff --git a/lang/js/test/dat/person-10.avro b/lang/js/test/dat/person-10.avro new file mode 100644 index 00000000000..9bff84de417 Binary files /dev/null and b/lang/js/test/dat/person-10.avro differ diff --git a/lang/js/test/dat/person-10.avro.raw b/lang/js/test/dat/person-10.avro.raw new file mode 100644 index 00000000000..64616339e9c Binary files /dev/null and b/lang/js/test/dat/person-10.avro.raw differ diff --git a/lang/js/test/dat/person-10.no-codec.avro b/lang/js/test/dat/person-10.no-codec.avro new file mode 100644 index 00000000000..e1b433b888f Binary files /dev/null and b/lang/js/test/dat/person-10.no-codec.avro differ diff --git a/lang/js/test/test_files.js b/lang/js/test/test_files.js new file mode 100644 index 00000000000..c0a334b81f1 --- /dev/null +++ b/lang/js/test/test_files.js @@ -0,0 +1,598 @@ +/* jshint node: true, mocha: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +var files = require('../lib/files'), + schemas = require('../lib/schemas'), + assert = require('assert'), + fs = require('fs'), + path = require('path'), + tmp = require('tmp'); + +var DPATH = path.join(__dirname, 'dat'); +var Header = files.HEADER_TYPE.getRecordConstructor(); +var MAGIC_BYTES = files.MAGIC_BYTES; +var SYNC = new Buffer('atokensyncheader'); +var createType = schemas.createType; +var streams = files.streams; +var types = schemas.types; + + +suite('files', function () { + + suite('parse', function () { + + var parse = files.parse; + + test('object', function () { + var obj = { + type: 'record', + name: 'Person', + fields: [{name: 'so', type: 'Person'}] + }; + assert(parse(obj) instanceof types.RecordType); + }); + + test('schema instance', function () { + var type = parse({ + type: 'record', + name: 'Person', + fields: [{name: 'so', type: 'Person'}] + }); + assert.strictEqual(parse(type), type); + }); + + test('stringified schema', function () { + assert(parse('"int"') instanceof types.IntType); + }); + + test('type name', function () { + assert(parse('double') instanceof types.DoubleType); + }); + + test('file', function () { + var t1 = parse({type: 'fixed', name: 'id.Id', size: 64}); + var t2 = parse(path.join(__dirname, 'dat', 'Id.avsc')); + assert.deepEqual(JSON.stringify(t1), JSON.stringify(t2)); + }); + + }); + + suite('RawEncoder', function () { + + var RawEncoder = streams.RawEncoder; + + test('flush once', function (cb) { + var t = createType('int'); + var buf; + var encoder = new RawEncoder(t) + .on('data', function (chunk) { + assert.strictEqual(buf, undefined); + buf = chunk; + }) + .on('end', function () { + assert.deepEqual(buf, new Buffer([2, 0, 3])); + cb(); + }); + encoder.write(1); + encoder.write(0); + encoder.end(-2); + }); + + test('write multiple', function (cb) { + var t = createType('int'); + var bufs = []; + var encoder = new RawEncoder(t, {batchSize: 1}) + .on('data', function (chunk) { + bufs.push(chunk); + }) + .on('end', function () { + assert.deepEqual(bufs, [new Buffer([1]), new Buffer([2])]); + cb(); + }); + encoder.write(-1); + encoder.end(1); + }); + + test('resize', function (cb) { + var t = createType({type: 'fixed', name: 'A', size: 2}); + var data = new Buffer([48, 18]); + var buf; + var encoder = new RawEncoder(t, {batchSize: 1}) + .on('data', function (chunk) { + assert.strictEqual(buf, undefined); + buf = chunk; + }) + .on('end', function () { + assert.deepEqual(buf, data); + cb(); + }); + encoder.write(data); + encoder.end(); + }); + + test('flush when full', function (cb) { + var t = createType({type: 'fixed', name: 'A', size: 2}); + var data = new Buffer([48, 18]); + var chunks = []; + var encoder = new RawEncoder(t, {batchSize: 2}) + .on('data', function (chunk) { chunks.push(chunk); }) + .on('end', function () { + assert.deepEqual(chunks, [data, data]); + cb(); + }); + encoder.write(data); + encoder.write(data); + encoder.end(); + }); + + test('empty', function (cb) { + var t = createType('int'); + var chunks = []; + var encoder = new RawEncoder(t, {batchSize: 2}) + .on('data', function (chunk) { chunks.push(chunk); }) + .on('end', function () { + assert.deepEqual(chunks, []); + cb(); + }); + encoder.end(); + }); + + test('missing writer type', function () { + assert.throws(function () { new RawEncoder(); }); + }); + + test('writer type from schema', function () { + var encoder = new RawEncoder('int'); + assert(encoder._type instanceof types.IntType); + }); + + test('invalid object', function (cb) { + var t = createType('int'); + var encoder = new RawEncoder(t) + .on('error', function () { cb(); }); + encoder.write('hi'); + }); + + }); + + suite('RawDecoder', function () { + + var RawDecoder = streams.RawDecoder; + + test('single item', function (cb) { + var t = createType('int'); + var objs = []; + var decoder = new RawDecoder(t) + .on('data', function (obj) { objs.push(obj); }) + .on('end', function () { + assert.deepEqual(objs, [0]); + cb(); + }); + decoder.end(new Buffer([0])); + }); + + test('no writer type', function () { + assert.throws(function () { new RawDecoder(); }); + }); + + test('decoding', function (cb) { + var t = createType('int'); + var objs = []; + var decoder = new RawDecoder(t) + .on('data', function (obj) { objs.push(obj); }) + .on('end', function () { + assert.deepEqual(objs, [1, 2]); + cb(); + }); + decoder.write(new Buffer([2])); + decoder.end(new Buffer([4])); + }); + + test('no decoding', function (cb) { + var t = createType('int'); + var bufs = [new Buffer([3]), new Buffer([124])]; + var objs = []; + var decoder = new RawDecoder(t, {decode: false}) + .on('data', function (obj) { objs.push(obj); }) + .on('end', function () { + assert.deepEqual(objs, bufs); + cb(); + }); + decoder.write(bufs[0]); + decoder.end(bufs[1]); + }); + + test('write partial', function (cb) { + var t = createType('bytes'); + var objs = []; + var decoder = new RawDecoder(t) + .on('data', function (obj) { objs.push(obj); }) + .on('end', function () { + assert.deepEqual(objs, [new Buffer([6])]); + cb(); + }); + decoder.write(new Buffer([2])); + // Let the first read go through (and return null). + process.nextTick(function () { decoder.end(new Buffer([6])); }); + }); + + }); + + suite('BlockEncoder', function () { + + var BlockEncoder = streams.BlockEncoder; + + test('invalid type', function () { + assert.throws(function () { new BlockEncoder(); }); + }); + + test('invalid codec', function (cb) { + var t = createType('int'); + var encoder = new BlockEncoder(t, {codec: 'foo'}) + .on('error', function () { cb(); }); + encoder.write(2); + }); + + test('invalid object', function (cb) { + var t = createType('int'); + var encoder = new BlockEncoder(t) + .on('error', function () { cb(); }); + encoder.write('hi'); + }); + + test('empty', function (cb) { + var t = createType('int'); + var chunks = []; + var encoder = new BlockEncoder(t) + .on('data', function (chunk) { chunks.push(chunk); }) + .on('end', function () { + assert.equal(chunks.length, 0); + cb(); + }); + encoder.end(); + }); + + test('flush on finish', function (cb) { + var t = createType('int'); + var chunks = []; + var encoder = new BlockEncoder(t, { + omitHeader: true, + syncMarker: SYNC + }).on('data', function (chunk) { chunks.push(chunk); }) + .on('end', function () { + assert.deepEqual(chunks, [ + new Buffer([6]), + new Buffer([6]), + new Buffer([24, 0, 8]), + SYNC + ]); + cb(); + }); + encoder.write(12); + encoder.write(0); + encoder.end(4); + }); + + test('flush when full', function (cb) { + var chunks = []; + var encoder = new BlockEncoder(createType('int'), { + omitHeader: true, + syncMarker: SYNC, + blockSize: 2 + }).on('data', function (chunk) { chunks.push(chunk); }) + .on('end', function () { + assert.deepEqual( + chunks, + [ + new Buffer([2]), new Buffer([2]), new Buffer([2]), SYNC, + new Buffer([2]), new Buffer([4]), new Buffer([128, 1]), SYNC + ] + ); + cb(); + }); + encoder.write(1); + encoder.end(64); + }); + + test('resize', function (cb) { + var t = createType({type: 'fixed', size: 8, name: 'Eight'}); + var buf = new Buffer('abcdefgh'); + var chunks = []; + var encoder = new BlockEncoder(t, { + omitHeader: true, + syncMarker: SYNC, + blockSize: 4 + }).on('data', function (chunk) { chunks.push(chunk); }) + .on('end', function () { + var b1 = new Buffer([4]); + var b2 = new Buffer([32]); + assert.deepEqual(chunks, [b1, b2, Buffer.concat([buf, buf]), SYNC]); + cb(); + }); + encoder.write(buf); + encoder.end(buf); + }); + + test('compression error', function (cb) { + var t = createType('int'); + var codecs = { + invalid: function (data, cb) { cb(new Error('ouch')); } + }; + var encoder = new BlockEncoder(t, {codec: 'invalid', codecs: codecs}) + .on('error', function () { cb(); }); + encoder.end(12); + }); + + test('write non-canonical schema', function (cb) { + var obj = {type: 'fixed', size: 2, name: 'Id', doc: 'An id.'}; + var id = new Buffer([1, 2]); + var ids = []; + var encoder = new BlockEncoder(obj); + var decoder = new streams.BlockDecoder() + .on('metadata', function (type, codec, header) { + var schema = JSON.parse(header.meta['avro.schema'].toString()); + assert.deepEqual(schema, obj); // Check that doc field not stripped. + }) + .on('data', function (id) { ids.push(id); }) + .on('end', function () { + assert.deepEqual(ids, [id]); + cb(); + }); + encoder.pipe(decoder); + encoder.end(id); + }); + + }); + + suite('BlockDecoder', function () { + + var BlockDecoder = streams.BlockDecoder; + + test('invalid magic bytes', function (cb) { + var decoder = new BlockDecoder() + .on('data', function () {}) + .on('error', function () { cb(); }); + decoder.write(new Buffer([0, 3, 2, 1])); // !== MAGIC_BYTES + decoder.write(new Buffer([0])); + decoder.end(SYNC); + }); + + test('invalid sync marker', function (cb) { + var decoder = new BlockDecoder() + .on('data', function () {}) + .on('error', function () { cb(); }); + var header = new Header( + MAGIC_BYTES, + { + 'avro.schema': new Buffer('"int"'), + 'avro.codec': new Buffer('null') + }, + SYNC + ); + decoder.write(header.$toBuffer()); + decoder.write(new Buffer([0, 0])); // Empty block. + decoder.end(new Buffer('alongerstringthansixteenbytes')); + }); + + test('missing codec', function (cb) { + var decoder = new BlockDecoder() + .on('data', function () {}) + .on('end', function () { cb(); }); + var header = new Header( + MAGIC_BYTES, + {'avro.schema': new Buffer('"int"')}, + SYNC + ); + decoder.end(header.$toBuffer()); + }); + + test('unknown codec', function (cb) { + var decoder = new BlockDecoder() + .on('data', function () {}) + .on('error', function () { cb(); }); + var header = new Header( + MAGIC_BYTES, + { + 'avro.schema': new Buffer('"int"'), + 'avro.codec': new Buffer('"foo"') + }, + SYNC + ); + decoder.end(header.$toBuffer()); + }); + + test('invalid schema', function (cb) { + var decoder = new BlockDecoder() + .on('data', function () {}) + .on('error', function () { cb(); }); + var header = new Header( + MAGIC_BYTES, + { + 'avro.schema': new Buffer('"int2"'), + 'avro.codec': new Buffer('null') + }, + SYNC + ); + decoder.end(header.$toBuffer()); + }); + + }); + + suite('encode & decode', function () { + + test('uncompressed int', function (cb) { + var t = createType('int'); + var objs = []; + var encoder = new streams.BlockEncoder(t); + var decoder = new streams.BlockDecoder() + .on('data', function (obj) { objs.push(obj); }) + .on('end', function () { + assert.deepEqual(objs, [12, 23, 48]); + cb(); + }); + encoder.pipe(decoder); + encoder.write(12); + encoder.write(23); + encoder.end(48); + }); + + test('uncompressed int non decoded', function (cb) { + var t = createType('int'); + var objs = []; + var encoder = new streams.BlockEncoder(t); + var decoder = new streams.BlockDecoder({decode: false}) + .on('data', function (obj) { objs.push(obj); }) + .on('end', function () { + assert.deepEqual(objs, [new Buffer([96])]); + cb(); + }); + encoder.pipe(decoder); + encoder.end(48); + }); + + test('deflated records', function (cb) { + var t = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'name', type: 'string'}, + {name: 'age', type: 'int'} + ] + }); + var Person = t.getRecordConstructor(); + var p1 = [ + new Person('Ann', 23), + new Person('Bob', 25) + ]; + var p2 = []; + var encoder = new streams.BlockEncoder(t, {codec: 'deflate'}); + var decoder = new streams.BlockDecoder() + .on('data', function (obj) { p2.push(obj); }) + .on('end', function () { + assert.deepEqual(p2, p1); + cb(); + }); + encoder.pipe(decoder); + var i, l; + for (i = 0, l = p1.length; i < l; i++) { + encoder.write(p1[i]); + } + encoder.end(); + }); + + test('decompression error', function (cb) { + var t = createType('int'); + var codecs = { + 'null': function (data, cb) { cb(new Error('ouch')); } + }; + var encoder = new streams.BlockEncoder(t, {codec: 'null'}); + var decoder = new streams.BlockDecoder({codecs: codecs}) + .on('error', function () { cb(); }); + encoder.pipe(decoder); + encoder.end(1); + }); + + test('decompression late read', function (cb) { + var chunks = []; + var encoder = new streams.BlockEncoder(createType('int')); + var decoder = new streams.BlockDecoder(); + encoder.pipe(decoder); + encoder.end(1); + decoder.on('data', function (chunk) { chunks.push(chunk); }) + .on('end', function () { + assert.deepEqual(chunks, [1]); + cb(); + }); + }); + + }); + + test('createFileDecoder', function (cb) { + var n = 0; + var type = loadSchema(path.join(DPATH, 'Person.avsc')); + files.createFileDecoder(path.join(DPATH, 'person-10.avro')) + .on('metadata', function (writerType) { + assert.equal(writerType.toString(), type.toString()); + }) + .on('data', function (obj) { + n++; + assert(type.isValid(obj)); + }) + .on('end', function () { + assert.equal(n, 10); + cb(); + }); + }); + + test('createFileEncoder', function (cb) { + var type = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'name', type: 'string'}, + {name: 'age', type: 'int'} + ] + }); + var path = tmp.fileSync().name; + var encoder = files.createFileEncoder(path, type); + encoder.write({name: 'Ann', age: 32}); + encoder.end({name: 'Bob', age: 33}); + var n = 0; + encoder.on('finish', function () { + files.createFileDecoder(path) + .on('data', function (obj) { + n++; + assert(type.isValid(obj)); + }) + .on('end', function () { + assert.equal(n, 2); + cb(); + }); + }); + }); + + test('extractFileHeader', function () { + var header; + var fpath = path.join(DPATH, 'person-10.avro'); + header = files.extractFileHeader(fpath); + assert(header !== null); + assert.equal(typeof header.meta['avro.schema'], 'object'); + header = files.extractFileHeader(fpath, {decode: false}); + assert(Buffer.isBuffer(header.meta['avro.schema'])); + header = files.extractFileHeader(fpath, {size: 2}); + assert.equal(typeof header.meta['avro.schema'], 'object'); + header = files.extractFileHeader(path.join(DPATH, 'person-10.avro.raw')); + assert(header === null); + header = files.extractFileHeader( + path.join(DPATH, 'person-10.no-codec.avro') + ); + assert(header !== null); + }); + +}); + +// Helpers. + +function loadSchema(path) { + return createType(JSON.parse(fs.readFileSync(path))); +} diff --git a/lang/js/test/test_schemas.js b/lang/js/test/test_schemas.js new file mode 100644 index 00000000000..e96f3678f37 --- /dev/null +++ b/lang/js/test/test_schemas.js @@ -0,0 +1,2459 @@ +/* jshint node: true, mocha: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +var utils = require('../lib/utils'), + schemas = require('../lib/schemas'), + assert = require('assert'), + util = require('util'); + +var Tap = utils.Tap; +var createType = schemas.createType; +var types = schemas.types; + + +suite('types', function () { + + suite('BooleanType', function () { + + var data = [ + { + valid: [true, false], + invalid: [null, 'hi', undefined, 1.5, 1e28, 123124123123213] + } + ]; + + testType(types.BooleanType, data); + + test('to JSON', function () { + var t = new types.BooleanType(); + assert.equal(t.toJSON(), 'boolean'); + }); + + test('compare buffers', function () { + var t = new types.BooleanType(); + var bt = t.toBuffer(true); + var bf = t.toBuffer(false); + assert.equal(t.compareBuffers(bt, bf), 1); + assert.equal(t.compareBuffers(bf, bt), -1); + assert.equal(t.compareBuffers(bt, bt), 0); + }); + + }); + + suite('IntType', function () { + + var data = [ + { + valid: [1, -3, 12314, 0, 1e9], + invalid: [null, 'hi', undefined, 1.5, 1e28, 123124123123213] + } + ]; + + testType(types.IntType, data); + + test('toBuffer int', function () { + + var type = createType('int'); + assert.equal(type.fromBuffer(new Buffer([0x80, 0x01])), 64); + assert(new Buffer([0]).equals(type.toBuffer(0))); + + }); + + test('resolve int > long', function () { + var intType = createType('int'); + var longType = createType('long'); + var buf = intType.toBuffer(123); + assert.equal( + longType.fromBuffer(buf, longType.createResolver(intType)), + 123 + ); + }); + + test('resolve int > [null, int]', function () { + var wt = createType('int'); + var rt = createType(['null', 'int']); + var buf = wt.toBuffer(123); + assert.deepEqual( + rt.fromBuffer(buf, rt.createResolver(wt)), + {'int': 123} + ); + }); + + test('resolve int > float', function () { + var wt = createType('int'); + var rt = createType('float'); + var buf = wt.toBuffer(123); + assert.deepEqual(rt.fromBuffer(buf, rt.createResolver(wt)), 123); + }); + + test('resolve int > double', function () { + var wt = createType('int'); + var rt = createType('double'); + var n = Math.pow(2, 30) + 1; + var buf = wt.toBuffer(n); + assert.deepEqual(rt.fromBuffer(buf, rt.createResolver(wt)), n); + }); + + test('toString', function () { + assert.equal(createType('int').toString(), '"int"'); + }); + + test('clone', function () { + var t = createType('int'); + assert.equal(t.clone(123), 123); + assert.throws(function () { t.clone(''); }); + }); + + test('resolve invalid', function () { + assert.throws(function () { getResolver('int', 'long'); }); + }); + + }); + + suite('LongType', function () { + + var data = [ + { + valid: [1, -3, 12314, 9007199254740990, 900719925474090], + invalid: [null, 'hi', undefined, 9007199254740991, 1.3, 1e67] + } + ]; + + testType(types.LongType, data); + + test('resolve invalid', function () { + assert.throws(function () { getResolver('long', 'double'); }); + }); + + test('resolve long > float', function () { + var t1 = createType('long'); + var t2 = createType('float'); + var n = 9007199254740990; // Number.MAX_SAFE_INTEGER - 1 + var buf = t1.toBuffer(n); + var f = t2.fromBuffer(buf, t2.createResolver(t1)); + assert(Math.abs(f - n) / n < 1e-7); + assert(t2.isValid(f)); + }); + + test('precision loss', function () { + var type = createType('long'); + var buf = new Buffer([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x20]); + assert.throws(function () { type.fromBuffer(buf); }); + }); + + test('using missing methods', function () { + assert.throws(function () { types.LongType.using(); }); + }); + + }); + + suite('StringType', function () { + + var data = [ + { + valid: ['', 'hi'], + invalid: [null, undefined, 1, 0] + } + ]; + + testType(types.StringType, data); + + test('fromBuffer string', function () { + var type = createType('string'); + var buf = new Buffer([0x06, 0x68, 0x69, 0x21]); + var s = 'hi!'; + assert.equal(type.fromBuffer(buf), s); + assert(buf.equals(type.toBuffer(s))); + }); + + test('toBuffer string', function () { + var type = createType('string'); + var buf = new Buffer([0x06, 0x68, 0x69, 0x21]); + assert(buf.equals(type.toBuffer('hi!', 1))); + }); + + test('resolve string > bytes', function () { + var stringT = createType('string'); + var bytesT = createType('bytes'); + var buf = stringT.toBuffer('\x00\x01'); + assert.deepEqual( + bytesT.fromBuffer(buf, bytesT.createResolver(stringT)), + new Buffer([0, 1]) + ); + }); + + test('encode resize', function () { + var t = createType('string'); + var s = 'hello'; + var b, pos; + b = new Buffer(2); + pos = t.encode(s, b); + assert(pos < 0); + b = new Buffer(2 - pos); + pos = t.encode(s, b); + assert(pos >= 0); + assert.equal(s, t.fromBuffer(b)); // Also checks exact length match. + }); + + }); + + suite('NullType', function () { + + var data = [ + { + schema: 'null', + valid: [null], + invalid: [0, 1, 'hi', undefined] + } + ]; + + testType(types.NullType, data); + + }); + + suite('FloatType', function () { + + var data = [ + { + valid: [1, -3, 123e7], + invalid: [null, 'hi', undefined], + check: function (a, b) { assert(floatEquals(a, b)); } + } + ]; + + testType(types.FloatType, data); + + test('compare buffer', function () { + var t = createType('float'); + var b1 = t.toBuffer(0.5); + assert.equal(t.compareBuffers(b1, b1), 0); + var b2 = t.toBuffer(-0.75); + assert.equal(t.compareBuffers(b1, b2), 1); + var b3 = t.toBuffer(175); + assert.equal(t.compareBuffers(b1, b3), -1); + }); + + test('resolver double > float', function () { + assert.throws(function () { getResolver('float', 'double'); }); + }); + + test('fromString', function () { + var t = createType('float'); + var f = t.fromString('3.1'); + assert(t.isValid(f)); + }); + + test('clone from double', function () { + var t = createType('float'); + var d = 3.1; + var f; + f = t.clone(d); + assert(t.isValid(f)); + }); + + }); + + suite('DoubleType', function () { + + var data = [ + { + valid: [1, -3.4, 12314e31, 5e37], + invalid: [null, 'hi', undefined], + check: function (a, b) { assert(floatEquals(a, b), '' + [a, b]); } + } + ]; + + testType(types.DoubleType, data); + + test('resolver string > double', function () { + assert.throws(function () { getResolver('double', 'string'); }); + }); + + test('compare buffer', function () { + var t = createType('double'); + var b1 = t.toBuffer(0.5); + assert.equal(t.compareBuffers(b1, b1), 0); + var b2 = t.toBuffer(-0.75); + assert.equal(t.compareBuffers(b1, b2), 1); + var b3 = t.toBuffer(175); + assert.equal(t.compareBuffers(b1, b3), -1); + }); + + }); + + suite('BytesType', function () { + + var data = [ + { + valid: [new Buffer(1), new Buffer('abc')], + invalid: [null, 'hi', undefined, 1, 0, -3.5] + } + ]; + + testType(types.BytesType, data); + + test('clone', function () { + var t = createType('bytes'); + var s = '\x01\x02'; + var buf = new Buffer(s); + var clone; + clone = t.clone(buf); + assert.deepEqual(clone, buf); + clone[0] = 0; + assert.equal(buf[0], 1); + assert.throws(function () { t.clone(s); }); + clone = t.clone(buf.toJSON(), {coerceBuffers: true}); + assert.deepEqual(clone, buf); + assert.throws(function () { t.clone(1, {coerceBuffers: true}); }); + }); + + test('fromString', function () { + var t = createType('bytes'); + var s = '\x01\x02'; + var buf = new Buffer(s); + var clone = t.fromString(JSON.stringify(s)); + assert.deepEqual(clone, buf); + }); + + test('compare', function () { + var t = createType('bytes'); + var b1 = t.toBuffer(new Buffer([0, 2])); + assert.equal(t.compareBuffers(b1, b1), 0); + var b2 = t.toBuffer(new Buffer([0, 2, 3])); + assert.equal(t.compareBuffers(b1, b2), -1); + var b3 = t.toBuffer(new Buffer([1])); + assert.equal(t.compareBuffers(b3, b1), 1); + }); + + }); + + suite('UnionType', function () { + + var data = [ + { + name: 'null & string', + schema: ['null', 'string'], + valid: [null, {string: 'hi'}], + invalid: ['null', undefined, {string: 1}], + check: assert.deepEqual + }, + { + name: 'qualified name', + schema: ['null', {type: 'fixed', name: 'a.B', size: 2}], + valid: [null, {'a.B': new Buffer(2)}], + invalid: [new Buffer(2)], + check: assert.deepEqual + }, + { + name: 'array int', + schema: ['int', {type: 'array', items: 'int'}], + valid: [{'int': 1}, {array: [1,3]}], + invalid: [null, 2, {array: ['a']}, [4], 2], + check: assert.deepEqual + }, + { + name: 'null', + schema: ['null'], + valid: [null], + invalid: [{array: ['a']}, [4], 'null'], + check: assert.deepEqual + } + ]; + + var schemas = [ + {}, + [], + ['null', 'null'], + ['null', {type: 'map', values: 'int'}, {type: 'map', values: 'long'}], + ['null', ['int', 'string']] + ]; + + testType(types.UnionType, data, schemas); + + test('getTypes', function () { + var t = createType(['null', 'int']); + assert.deepEqual(t.getTypes(), [createType('null'), createType('int')]); + }); + + test('instanceof Union', function () { + var type = new types.UnionType(['null', 'int']); + assert(type instanceof types.UnionType); + }); + + test('missing name write', function () { + var type = new types.UnionType(['null', 'int']); + assert.throws(function () { + type.toBuffer({b: 'a'}); + }); + }); + + test('read invalid index', function () { + var type = new types.UnionType(['null', 'int']); + var buf = new Buffer([1, 0]); + assert.throws(function () { type.fromBuffer(buf); }); + }); + + test('non wrapped write', function () { + var type = new types.UnionType(['null', 'int']); + assert.throws(function () { + type.toBuffer(1, true); + }, Error); + }); + + test('to JSON', function () { + var type = new types.UnionType(['null', 'int']); + assert.equal(JSON.stringify(type), '["null","int"]'); + }); + + test('resolve int to [long, int]', function () { + var t1 = createType('int'); + var t2 = createType(['long', 'int']); + var a = t2.createResolver(t1); + var buf = t1.toBuffer(23); + assert.deepEqual(t2.fromBuffer(buf, a), {'long': 23}); + }); + + test('resolve null to [null, int]', function () { + var t1 = createType('null'); + var t2 = createType(['null', 'int']); + var a = t2.createResolver(t1); + assert.deepEqual(t2.fromBuffer(new Buffer(0), a), null); + }); + + test('resolve [string, int] to [long, string]', function () { + var t1 = createType(['string', 'int']); + var t2 = createType(['int', 'bytes']); + var a = t2.createResolver(t1); + var buf; + buf = t1.toBuffer({string: 'hi'}); + assert.deepEqual(t2.fromBuffer(buf, a), {'bytes': new Buffer('hi')}); + buf = t1.toBuffer({'int': 1}); + assert.deepEqual(t2.fromBuffer(buf, a), {'int': 1}); + }); + + test('clone', function () { + var t = new types.UnionType(['null', 'int']); + var o = {'int': 1}; + assert.strictEqual(t.clone(null), null); + var c = t.clone(o); + assert.deepEqual(c, o); + c.int = 2; + assert.equal(o.int, 1); + assert.throws(function () { t.clone([]); }); + assert.throws(function () { t.clone(undefined); }); + }); + + test('clone and wrap', function () { + var t = createType(['string', 'int']); + var o; + o = t.clone('hi', {wrapUnions: true}); + assert.deepEqual(o, {'string': 'hi'}); + o = t.clone(3, {wrapUnions: true}); + assert.deepEqual(o, {'int': 3}); + assert.throws(function () { t.clone(null, {wrapUnions: 2}); }); + }); + + test('invalid multiple keys', function () { + var t = createType(['null', 'int']); + var o = {'int': 2}; + assert(t.isValid(o)); + o.foo = 3; + assert(!t.isValid(o)); + }); + + test('clone multiple keys', function () { + var t = createType(['null', 'int']); + var o = {'int': 2, foo: 3}; + assert.throws(function () { t.clone(o); }); + }); + + test('clone unqualified names', function () { + var t = createType({ + name: 'Person', + type: 'record', + fields: [ + {name: 'id1', type: {name: 'an.Id', type: 'fixed', size: 1}}, + {name: 'id2', type: ['null', 'an.Id']} + ] + }); + var b = new Buffer([0]); + var o = {id1: b, id2: {Id: b}}; + assert.deepEqual(t.clone(o), {id1: b, id2: {'an.Id': b}}); + }); + + test('clone unqualified names', function () { + var t = createType({ + name: 'Person', + type: 'record', + fields: [ + {name: 'id1', type: {name: 'Id', type: 'fixed', size: 1}}, + {name: 'id2', type: ['null', 'Id']} + ] + }); + var b = new Buffer([0]); + var o = {id1: b, id2: {'an.Id': b}}; + assert.throws(function () { t.clone(o); }); + }); + + test('compare buffers', function () { + var t = createType(['null', 'double']); + var b1 = t.toBuffer(null); + assert.equal(t.compareBuffers(b1, b1), 0); + var b2 = t.toBuffer({'double': 4}); + assert.equal(t.compareBuffers(b2, b1), 1); + assert.equal(t.compareBuffers(b1, b2), -1); + var b3 = t.toBuffer({'double': 6}); + assert.equal(t.compareBuffers(b3, b2), 1); + }); + + test('compare', function () { + var t; + t = createType(['null', 'int']); + assert.equal(t.compare(null, {'int': 3}), -1); + assert.equal(t.compare(null, null), 0); + t = createType(['int', 'float']); + assert.equal(t.compare({'int': 2}, {'float': 0.5}), -1); + assert.equal(t.compare({'int': 20}, {'int': 5}), 1); + }); + + }); + + suite('EnumType', function () { + + var data = [ + { + name: 'single symbol', + schema: {name: 'Foo', symbols: ['HI']}, + valid: ['HI'], + invalid: ['HEY', null, undefined, 0] + }, + { + name: 'number-ish as symbol', + schema: {name: 'Foo', symbols: ['HI', 'A0']}, + valid: ['HI', 'A0'], + invalid: ['HEY', null, undefined, 0, 'a0'] + } + ]; + + var schemas = [ + {name: 'Foo', symbols: []}, + {name: 'Foo'}, + {symbols: ['hi']}, + {name: 'G', symbols: ['0']} + ]; + + testType(types.EnumType, data, schemas); + + test('get full name', function () { + var t = createType({ + type: 'enum', + symbols: ['A', 'B'], + name: 'Letter', + namespace: 'latin' + }); + assert.equal(t.getName(), 'latin.Letter'); + }); + + test('get aliases', function () { + var t = createType({ + type: 'enum', + symbols: ['A', 'B'], + name: 'Letter', + namespace: 'latin', + aliases: ['Character', 'alphabet.Letter'] + }); + var aliases = t.getAliases(); + assert.deepEqual(aliases, ['latin.Character', 'alphabet.Letter']); + aliases.push('Char'); + assert.equal(t.getAliases().length, 3); + }); + + test('get symbols', function () { + var t = createType({type: 'enum', symbols: ['A', 'B'], name: 'Letter'}); + var symbols = t.getSymbols(); + assert.deepEqual(symbols, ['A', 'B']); + symbols.push('Char'); + assert.equal(t.getSymbols().length, 2); + }); + + test('duplicate symbol', function () { + assert.throws(function () { + createType({type: 'enum', symbols: ['A', 'B', 'A'], name: 'B'}); + }); + }); + + test('write invalid', function () { + var type = createType({type: 'enum', symbols: ['A'], name: 'a'}); + assert.throws(function () { + type.toBuffer('B'); + }); + }); + + test('read invalid index', function () { + var type = new types.EnumType({type: 'enum', symbols: ['A'], name: 'a'}); + var buf = new Buffer([2]); + assert.throws(function () { type.fromBuffer(buf); }); + }); + + test('resolve', function () { + var t1, t2, buf, resolver; + t1 = newEnum('Foo', ['bar', 'baz']); + t2 = newEnum('Foo', ['bar', 'baz']); + resolver = t1.createResolver(t2); + buf = t2.toBuffer('bar'); + assert.equal(t1.fromBuffer(buf, resolver), 'bar'); + t2 = newEnum('Foo', ['baz', 'bar']); + buf = t2.toBuffer('bar'); + resolver = t1.createResolver(t2); + assert.notEqual(t1.fromBuffer(buf), 'bar'); + assert.equal(t1.fromBuffer(buf, resolver), 'bar'); + t1 = newEnum('Foo2', ['foo', 'baz', 'bar'], ['Foo']); + resolver = t1.createResolver(t2); + assert.equal(t1.fromBuffer(buf, resolver), 'bar'); + t2 = newEnum('Foo', ['bar', 'bax']); + assert.throws(function () { t1.createResolver(t2); }); + assert.throws(function () { + t1.createResolver(createType('int')); + }); + function newEnum(name, symbols, aliases, namespace) { + var obj = {type: 'enum', name: name, symbols: symbols}; + if (aliases !== undefined) { + obj.aliases = aliases; + } + if (namespace !== undefined) { + obj.namespace = namespace; + } + return new types.EnumType(obj); + } + }); + + test('clone', function () { + var t = createType({type: 'enum', name: 'Foo', symbols: ['bar', 'baz']}); + assert.equal(t.clone('bar'), 'bar'); + assert.throws(function () { t.clone('BAR'); }); + assert.throws(function () { t.clone(null); }); + }); + + test('compare buffers', function () { + var t = createType({type: 'enum', name: 'Foo', symbols: ['bar', 'baz']}); + var b1 = t.toBuffer('bar'); + var b2 = t.toBuffer('baz'); + assert.equal(t.compareBuffers(b1, b1), 0); + assert.equal(t.compareBuffers(b2, b1), 1); + }); + + test('compare', function () { + var t = createType({type: 'enum', name: 'Foo', symbols: ['b', 'a']}); + assert.equal(t.compare('b', 'a'), -1); + assert.equal(t.compare('a', 'a'), 0); + }); + + }); + + suite('FixedType', function () { + + var data = [ + { + name: 'size 1', + schema: {name: 'Foo', size: 2}, + valid: [new Buffer([1, 2]), new Buffer([2, 3])], + invalid: ['HEY', null, undefined, 0, new Buffer(1), new Buffer(3)], + check: function (a, b) { assert(a.equals(b)); } + } + ]; + + var schemas = [ + {name: 'Foo', size: 0}, + {name: 'Foo', size: -2}, + {size: 2}, + {name: 'Foo'}, + {} + ]; + + testType(types.FixedType, data, schemas); + + test('get full name', function () { + var t = createType({ + type: 'fixed', + size: 2, + name: 'Id', + namespace: 'id' + }); + assert.equal(t.getName(), 'id.Id'); + }); + + test('get aliases', function () { + var t = createType({ + type: 'fixed', + size: 3, + name: 'Id' + }); + var aliases = t.getAliases(); + assert.deepEqual(aliases, []); + aliases.push('ID'); + assert.equal(t.getAliases().length, 1); + }); + + test('get size', function () { + var t = createType({type: 'fixed', size: 5, name: 'Id'}); + assert.equal(t.getSize(), 5); + }); + + test('resolve', function () { + var t1 = new types.FixedType({name: 'Id', size: 4}); + var t2 = new types.FixedType({name: 'Id', size: 4}); + assert.doesNotThrow(function () { t2.createResolver(t1); }); + t2 = new types.FixedType({name: 'Id2', size: 4}); + assert.throws(function () { t2.createResolver(t1); }); + t2 = new types.FixedType({name: 'Id2', size: 4, aliases: ['Id']}); + assert.doesNotThrow(function () { t2.createResolver(t1); }); + t2 = new types.FixedType({name: 'Id2', size: 5, aliases: ['Id']}); + assert.throws(function () { t2.createResolver(t1); }); + }); + + test('clone', function () { + var t = new types.FixedType({name: 'Id', size: 2}); + var s = '\x01\x02'; + var buf = new Buffer(s); + var clone; + clone = t.clone(buf); + assert.deepEqual(clone, buf); + clone[0] = 0; + assert.equal(buf[0], 1); + assert.throws(function () { t.clone(s); }); + clone = t.clone(buf.toJSON(), {coerceBuffers: true}); + assert.deepEqual(clone, buf); + assert.throws(function () { t.clone(1, {coerceBuffers: true}); }); + assert.throws(function () { t.clone(new Buffer([2])); }); + }); + + test('getSchema with extra fields', function () { + var t = createType({type: 'fixed', name: 'Id', size: 2, three: 3}); + t.one = 1; + assert.equal(t.getSchema(), '{"name":"Id","type":"fixed","size":2}'); + assert.equal(t.getSchema(true), '"Id"'); + }); + + test('fromString', function () { + var t = new types.FixedType({name: 'Id', size: 2}); + var s = '\x01\x02'; + var buf = new Buffer(s); + var clone = t.fromString(JSON.stringify(s)); + assert.deepEqual(clone, buf); + }); + + test('compare buffers', function () { + var t = createType({type: 'fixed', name: 'Id', size: 2}); + var b1 = new Buffer([1, 2]); + assert.equal(t.compareBuffers(b1, b1), 0); + var b2 = new Buffer([2, 2]); + assert.equal(t.compareBuffers(b1, b2), -1); + }); + + }); + + suite('MapType', function () { + + var data = [ + { + name: 'int', + schema: {values: 'int'}, + valid: [{one: 1}, {two: 2, o: 0}], + invalid: [1, {o: null}, [], undefined, {o: 'hi'}, {1: '', 2: 3}, ''], + check: assert.deepEqual + }, + { + name: 'enum', + schema: {values: {type: 'enum', name: 'a', symbols: ['A', 'B']}}, + valid: [{a: 'A'}, {a: 'A', b: 'B'}, {}], + invalid: [{o: 'a'}, {1: 'A', 2: 'b'}, {a: 3}], + check: assert.deepEqual + }, + { + name: 'array of string', + schema: {values: {type: 'array', items: 'string'}}, + valid: [{a: []}, {a: ['A'], b: ['B', '']}, {}], + invalid: [{o: 'a', b: []}, {a: [1, 2]}, {a: {b: ''}}], + check: assert.deepEqual + } + ]; + + var schemas = [ + {}, + {values: ''}, + {values: {type: 'array'}} + ]; + + testType(types.MapType, data, schemas); + + test('get values type', function () { + var t = new types.MapType({type: 'map', values: 'int'}); + assert.deepEqual(t.getValuesType(), createType('int')); + }); + + test('write int', function () { + var t = new types.MapType({type: 'map', values: 'int'}); + var buf = t.toBuffer({'\x01': 3, '\x02': 4}); + assert.deepEqual(buf, new Buffer([4, 2, 1, 6, 2, 2, 8, 0])); + }); + + test('read long', function () { + var t = new types.MapType({type: 'map', values: 'long'}); + var buf = new Buffer([4, 2, 1, 6, 2, 2, 8, 0]); + assert.deepEqual(t.fromBuffer(buf), {'\x01': 3, '\x02': 4}); + }); + + test('read with sizes', function () { + var t = new types.MapType({type: 'map', values: 'int'}); + var buf = new Buffer([1,6,2,97,2,0]); + assert.deepEqual(t.fromBuffer(buf), {a: 1}); + }); + + test('skip', function () { + var v1 = createType({ + name: 'Foo', + type: 'record', + fields: [ + {name: 'map', type: {type: 'map', values: 'int'}}, + {name: 'val', type: 'int'} + ] + }); + var v2 = createType({ + name: 'Foo', + type: 'record', + fields: [{name: 'val', type: 'int'}] + }); + var b1 = new Buffer([2,2,97,2,0,6]); // Without sizes. + var b2 = new Buffer([1,6,2,97,2,0,6]); // With sizes. + var resolver = v2.createResolver(v1); + assert.deepEqual(v2.fromBuffer(b1, resolver), {val: 3}); + assert.deepEqual(v2.fromBuffer(b2, resolver), {val: 3}); + }); + + test('resolve int > long', function () { + var t1 = new types.MapType({type: 'map', values: 'int'}); + var t2 = new types.MapType({type: 'map', values: 'long'}); + var resolver = t2.createResolver(t1); + var obj = {one: 1, two: 2}; + var buf = t1.toBuffer(obj); + assert.deepEqual(t2.fromBuffer(buf, resolver), obj); + }); + + test('resolve double > double', function () { + var t = new types.MapType({type: 'map', values: 'double'}); + var resolver = t.createResolver(t); + var obj = {one: 1, two: 2}; + var buf = t.toBuffer(obj); + assert.deepEqual(t.fromBuffer(buf, resolver), obj); + }); + + test('resolve invalid', function () { + var t1 = new types.MapType({type: 'map', values: 'int'}); + var t2 = new types.MapType({type: 'map', values: 'string'}); + assert.throws(function () { t2.createResolver(t1); }); + t2 = new types.ArrayType({type: 'array', items: 'string'}); + assert.throws(function () { t2.createResolver(t1); }); + }); + + test('resolve fixed', function () { + var t1 = createType({ + type: 'map', values: {name: 'Id', type: 'fixed', size: 2} + }); + var t2 = createType({ + type: 'map', values: { + name: 'Id2', aliases: ['Id'], type: 'fixed', size: 2 + } + }); + var resolver = t2.createResolver(t1); + var obj = {one: new Buffer([1, 2])}; + var buf = t1.toBuffer(obj); + assert.deepEqual(t2.fromBuffer(buf, resolver), obj); + }); + + test('clone', function () { + var t = new types.MapType({type: 'map', values: 'int'}); + var o = {one: 1, two: 2}; + var c = t.clone(o); + assert.deepEqual(c, o); + c.one = 3; + assert.equal(o.one, 1); + assert.throws(function () { t.clone(undefined); }); + }); + + test('clone coerce buffers', function () { + var t = new types.MapType({type: 'map', values: 'bytes'}); + var o = {one: {type: 'Buffer', data: [1]}}; + assert.throws(function () { t.clone(o); }); + var c = t.clone(o, {coerceBuffers: true}); + assert.deepEqual(c, {one: new Buffer([1])}); + }); + + test('compare buffers', function () { + var t = new types.MapType({type: 'map', values: 'bytes'}); + var b1 = t.toBuffer({}); + assert.throws(function () { t.compareBuffers(b1, b1); }); + }); + + test('isValid hook', function () { + var t = new types.MapType({type: 'map', values: 'int'}); + var o = {one: 1, two: 'deux', three: null, four: 4}; + var errs = {}; + assert(!t.isValid(o, {errorHook: hook})); + assert.deepEqual(errs, {two: 'deux', three: null}); + + function hook(path, obj, type) { + assert.strictEqual(type, t.getValuesType()); + assert.equal(path.length, 1); + errs[path[0]] = obj; + } + }); + + test('getName', function () { + var t = new types.MapType({type: 'map', values: 'int'}); + assert.strictEqual(t.getName(), undefined); + }); + + }); + + suite('ArrayType', function () { + + var data = [ + { + name: 'int', + schema: {items: 'int'}, + valid: [[1,3,4], []], + invalid: [1, {o: null}, undefined, ['a'], [true]], + check: assert.deepEqual + } + ]; + + var schemas = [ + {}, + {items: ''}, + ]; + + testType(types.ArrayType, data, schemas); + + test('get items type', function () { + var t = new types.ArrayType({type: 'array', items: 'int'}); + assert.deepEqual(t.getItemsType(), createType('int')); + }); + + test('read with sizes', function () { + var t = new types.ArrayType({type: 'array', items: 'int'}); + var buf = new Buffer([1,2,2,0]); + assert.deepEqual(t.fromBuffer(buf), [1]); + }); + + test('skip', function () { + var v1 = createType({ + name: 'Foo', + type: 'record', + fields: [ + {name: 'array', type: {type: 'array', items: 'int'}}, + {name: 'val', type: 'int'} + ] + }); + var v2 = createType({ + name: 'Foo', + type: 'record', + fields: [{name: 'val', type: 'int'}] + }); + var b1 = new Buffer([2,2,0,6]); // Without sizes. + var b2 = new Buffer([1,2,2,0,6]); // With sizes. + var resolver = v2.createResolver(v1); + assert.deepEqual(v2.fromBuffer(b1, resolver), {val: 3}); + assert.deepEqual(v2.fromBuffer(b2, resolver), {val: 3}); + }); + + test('resolve string items to bytes items', function () { + var t1 = new types.ArrayType({type: 'array', items: 'string'}); + var t2 = new types.ArrayType({type: 'array', items: 'bytes'}); + var resolver = t2.createResolver(t1); + var obj = ['\x01\x02']; + var buf = t1.toBuffer(obj); + assert.deepEqual(t2.fromBuffer(buf, resolver), [new Buffer([1, 2])]); + }); + + test('resolve invalid', function () { + var t1 = new types.ArrayType({type: 'array', items: 'string'}); + var t2 = new types.ArrayType({type: 'array', items: 'long'}); + assert.throws(function () { t2.createResolver(t1); }); + t2 = new types.MapType({type: 'map', values: 'string'}); + assert.throws(function () { t2.createResolver(t1); }); + }); + + test('clone', function () { + var t = new types.ArrayType({type: 'array', items: 'int'}); + var o = [1, 2]; + var c = t.clone(o); + assert.deepEqual(c, o); + c.one = 3; + assert.equal(o[0], 1); + assert.throws(function () { t.clone({}); }); + }); + + test('clone coerce buffers', function () { + var t = createType({ + type: 'array', + items: {type: 'fixed', name: 'Id', size: 2} + }); + var o = [{type: 'Buffer', data: [1, 2]}]; + assert.throws(function () { t.clone(o); }); + var c = t.clone(o, {coerceBuffers: true}); + assert.deepEqual(c, [new Buffer([1, 2])]); + }); + + test('compare buffers', function () { + var t = createType({type: 'array', items: 'int'}); + assert.equal(t.compareBuffers(t.toBuffer([]), t.toBuffer([])), 0); + assert.equal(t.compareBuffers(t.toBuffer([1, 2]), t.toBuffer([])), 1); + assert.equal(t.compareBuffers(t.toBuffer([1]), t.toBuffer([1, -1])), -1); + assert.equal(t.compareBuffers(t.toBuffer([1]), t.toBuffer([2])), -1); + assert.equal(t.compareBuffers(t.toBuffer([1, 2]), t.toBuffer([1])), 1); + }); + + test('compare', function () { + var t = createType({type: 'array', items: 'int'}); + assert.equal(t.compare([], []), 0); + assert.equal(t.compare([], [-1]), -1); + assert.equal(t.compare([1], [1]), 0); + assert.equal(t.compare([2], [1, 2]), 1); + }); + + test('isValid hook invalid array', function () { + var t = createType({type: 'array', items: 'int'}); + var hookCalled = false; + assert(!t.isValid({}, {errorHook: hook})); + assert(hookCalled); + + function hook(path, obj, type) { + assert.strictEqual(type, t); + assert.deepEqual(path, []); + hookCalled = true; + } + }); + + test('isValid hook invalid elems', function () { + var t = createType({type: 'array', items: 'int'}); + var paths = []; + assert(!t.isValid([0, 3, 'hi', 5, 'hey'], {errorHook: hook})); + assert.deepEqual(paths, [['2'], ['4']]); + + function hook(path, obj, type) { + assert.strictEqual(type, t.getItemsType()); + assert.equal(typeof obj, 'string'); + paths.push(path); + } + }); + + }); + + suite('RecordType', function () { + + var data = [ + { + name: 'union field null and string with default', + schema: { + type: 'record', + name: 'a', + fields: [{name: 'b', type: ['null', 'string'], 'default': null}] + }, + valid: [], + invalid: [], + check: assert.deepEqual + } + ]; + + var schemas = [ + {type: 'record', name: 'a', fields: ['null', 'string']}, + {type: 'record', name: 'a', fields: [{type: ['null', 'string']}]}, + { + type: 'record', + name: 'a', + fields: [{name: 'b', type: ['null', 'string'], 'default': 'a'}] + }, + {type: 'record', name: 'a', fields: {type: 'int', name: 'age'}} + ]; + + testType(types.RecordType, data, schemas); + + test('duplicate field names', function () { + assert.throws(function () { + createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}, {name: 'age', type: 'float'}] + }); + }); + }); + + test('default constructor', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int', 'default': 25}] + }); + var Person = type.getRecordConstructor(); + var p = new Person(); + assert.equal(p.age, 25); + assert.strictEqual(p.constructor, Person); + }); + + test('default check & write', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'age', type: 'int', 'default': 25}, + {name: 'name', type: 'string', 'default': '\x01'} + ] + }); + assert.deepEqual(type.toBuffer({}), new Buffer([50, 2, 1])); + }); + + test('fixed string default', function () { + var s = '\x01\x04'; + var b = new Buffer(s); + var type = createType({ + type: 'record', + name: 'Object', + fields: [ + { + name: 'id', + type: {type: 'fixed', size: 2, name: 'Id'}, + 'default': s + } + ] + }); + var obj = new (type.getRecordConstructor())(); + assert.deepEqual(obj.id, new Buffer([1, 4])); + assert.deepEqual(type.toBuffer({}), b); + }); + + test('fixed buffer invalid default', function () { + assert.throws(function () { + createType({ + type: 'record', + name: 'Object', + fields: [ + { + name: 'id', + type: {type: 'fixed', size: 2, name: 'Id'}, + 'default': new Buffer([0]) + } + ] + }); + }); + }); + + test('union invalid default', function () { + assert.throws(function () { + createType({ + type: 'record', + name: 'Person', + fields: [{name: 'name', type: ['null', 'string'], 'default': ''}] + }); + }); + }); + + test('record default', function () { + var d = {street: null, zip: 123}; + var Person = createType({ + name: 'Person', + type: 'record', + fields: [ + { + name: 'address', + type: { + name: 'Address', + type: 'record', + fields: [ + {name: 'street', type: ['null', 'string']}, + {name: 'zip', type: ['int', 'string']} + ] + }, + 'default': d + } + ] + }).getRecordConstructor(); + var p = new Person(); + assert.deepEqual(p.address, {street: null, zip: {'int': 123}}); + }); + + test('record keyword field name', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'null', type: 'int'}] + }); + var Person = type.getRecordConstructor(); + assert.deepEqual(new Person(2), {'null': 2}); + }); + + test('record isValid', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}] + }); + var Person = type.getRecordConstructor(); + assert((new Person(20)).$isValid()); + assert(!(new Person()).$isValid()); + assert(!(new Person('a')).$isValid()); + }); + + test('record toBuffer', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}] + }); + var Person = type.getRecordConstructor(); + assert.deepEqual((new Person(48)).$toBuffer(), new Buffer([96])); + assert.throws(function () { (new Person()).$toBuffer(); }); + }); + + test('record compare', function () { + var P = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'data', type: {type: 'map', values: 'int'}, order:'ignore'}, + {name: 'age', type: 'int'} + ] + }).getRecordConstructor(); + var p1 = new P({}, 1); + var p2 = new P({}, 2); + assert.equal(p1.$compare(p2), -1); + assert.equal(p2.$compare(p2), 0); + assert.equal(p2.$compare(p1), 1); + }); + + test('Record type', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}] + }); + var Person = type.getRecordConstructor(); + assert.strictEqual(Person.getType(), type); + }); + + test('mutable defaults', function () { + var Person = createType({ + type: 'record', + name: 'Person', + fields: [ + { + name: 'friends', + type: {type: 'array', items: 'string'}, + 'default': [] + } + ] + }).getRecordConstructor(); + var p1 = new Person(undefined); + assert.deepEqual(p1.friends, []); + p1.friends.push('ann'); + var p2 = new Person(undefined); + assert.deepEqual(p2.friends, []); + }); + + test('resolve alias', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'name', type: 'string'}] + }); + var p = v1.random(); + var buf = v1.toBuffer(p); + var v2 = createType({ + type: 'record', + name: 'Human', + aliases: ['Person'], + fields: [{name: 'name', type: 'string'}] + }); + var resolver = v2.createResolver(v1); + assert.deepEqual(v2.fromBuffer(buf, resolver), p); + var v3 = createType({ + type: 'record', + name: 'Human', + fields: [{name: 'name', type: 'string'}] + }); + assert.throws(function () { v3.createResolver(v1); }); + }); + + test('resolve alias with namespace', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + namespace: 'earth', + fields: [{name: 'name', type: 'string'}] + }); + var v2 = createType({ + type: 'record', + name: 'Human', + aliases: ['Person'], + fields: [{name: 'name', type: 'string'}] + }); + assert.throws(function () { v2.createResolver(v1); }); + var v3 = createType({ + type: 'record', + name: 'Human', + aliases: ['earth.Person'], + fields: [{name: 'name', type: 'string'}] + }); + assert.doesNotThrow(function () { v3.createResolver(v1); }); + }); + + test('resolve skip field', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'age', type: 'int'}, + {name: 'name', type: 'string'} + ] + }); + var p = {age: 25, name: 'Ann'}; + var buf = v1.toBuffer(p); + var v2 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'name', type: 'string'}] + }); + var resolver = v2.createResolver(v1); + assert.deepEqual(v2.fromBuffer(buf, resolver), {name: 'Ann'}); + }); + + test('resolve new field', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'name', type: 'string'}] + }); + var p = {name: 'Ann'}; + var buf = v1.toBuffer(p); + var v2 = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'age', type: 'int', 'default': 25}, + {name: 'name', type: 'string'} + ] + }); + var resolver = v2.createResolver(v1); + assert.deepEqual(v2.fromBuffer(buf, resolver), {name: 'Ann', age: 25}); + }); + + test('resolve new field no default', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'name', type: 'string'}] + }); + var v2 = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'age', type: 'int'}, + {name: 'name', type: 'string'} + ] + }); + assert.throws(function () { v2.createResolver(v1); }); + }); + + test('resolve from recursive schema', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'friends', type: {type: 'array', items: 'Person'}}] + }); + var v2 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int', 'default': -1}] + }); + var resolver = v2.createResolver(v1); + var p1 = {friends: [{friends: []}]}; + var p2 = v2.fromBuffer(v1.toBuffer(p1), resolver); + assert.deepEqual(p2, {age: -1}); + }); + + test('resolve to recursive schema', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int', 'default': -1}] + }); + var v2 = createType({ + type: 'record', + name: 'Person', + fields: [ + { + name: 'friends', + type: {type: 'array', items: 'Person'}, + 'default': [] + } + ] + }); + var resolver = v2.createResolver(v1); + var p1 = {age: 25}; + var p2 = v2.fromBuffer(v1.toBuffer(p1), resolver); + assert.deepEqual(p2, {friends: []}); + }); + + test('resolve from both recursive schema', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'friends', type: {type: 'array', items: 'Person'}}, + {name: 'age', type: 'int'} + ] + }); + var v2 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'friends', type: {type: 'array', items: 'Person'}}] + }); + var resolver = v2.createResolver(v1); + var p1 = {friends: [{age: 1, friends: []}], age: 10}; + var p2 = v2.fromBuffer(v1.toBuffer(p1), resolver); + assert.deepEqual(p2, {friends: [{friends: []}]}); + }); + + test('resolve multiple matching aliases', function () { + var v1 = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'phone', type: 'string'}, + {name: 'number', type: 'string'} + ] + }); + var v2 = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'number', type: 'string', aliases: ['phone']}] + }); + assert.throws(function () { v2.createResolver(v1); }); + }); + + test('getSchema', function () { + var t = createType({ + type: 'record', + name: 'Person', + doc: 'Hi!', + namespace: 'earth', + aliases: ['Human'], + fields: [ + {name: 'friends', type: {type: 'array', items: 'string'}}, + {name: 'age', aliases: ['years'], type: {type: 'int'}} + ] + }); + assert.equal( + t.getSchema(), + '{"name":"earth.Person","type":"record","fields":[{"name":"friends","type":{"type":"array","items":"string"}},{"name":"age","type":"int"}]}' + ); + assert.equal(t.getSchema(true), '"earth.Person"'); + }); + + test('getSchema recursive schema', function () { + var t = createType({ + type: 'record', + name: 'Person', + namespace: 'earth', + fields: [ + {name: 'friends', type: {type: 'array', items: 'Person'}}, + ] + }); + assert.equal( + t.getSchema(), + '{"name":"earth.Person","type":"record","fields":[{"name":"friends","type":{"type":"array","items":"earth.Person"}}]}' + ); + assert.equal(t.getSchema(true), '"earth.Person"'); + }); + + test('toString record', function () { + var T = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'pwd', type: 'bytes'}] + }).getRecordConstructor(); + var r = new T(new Buffer([1, 2])); + assert.equal(r.$toString(), T.getType().toString(r)); + }); + + test('clone', function () { + var t = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}, {name: 'name', type: 'string'}] + }); + var Person = t.getRecordConstructor(); + var o = {age: 25, name: 'Ann'}; + var c = t.clone(o); + assert.deepEqual(c, o); + assert(c instanceof Person); + c.age = 26; + assert.equal(o.age, 25); + assert.strictEqual(c.$getType(), t); + assert.deepEqual(c.$clone(), c); + }); + + test('clone field hook', function () { + var t = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}, {name: 'name', type: 'string'}] + }); + var o = {name: 'Ann', age: 25}; + var c = t.clone(o, {fieldHook: function (f, o, r) { + assert.strictEqual(r, t); + return f._type instanceof types.StringType ? o.toUpperCase() : o; + }}); + assert.deepEqual(c, {name: 'ANN', age: 25}); + }); + + test('get full name & aliases', function () { + var t = createType({ + type: 'record', + name: 'Person', + namespace: 'a', + fields: [{name: 'age', type: 'int'}, {name: 'name', type: 'string'}] + }); + assert.equal(t.getName(), 'a.Person'); + assert.deepEqual(t.getAliases(), []); + }); + + test('field getters', function () { + var t = createType({ + type: 'record', + name: 'Person', + namespace: 'a', + fields: [ + {name: 'age', type: 'int'}, + {name: 'name', type: 'string', aliases: ['word'], namespace: 'b'} + ] + }); + var fields = t.getFields(); + assert.deepEqual(fields[0].getAliases(), []); + assert.deepEqual(fields[1].getAliases(), ['word']); + assert.equal(fields[1].getName(), 'name'); // Namespaces are ignored. + assert.deepEqual(fields[1].getType(), createType('string')); + fields.push('null'); + assert.equal(t.getFields().length, 2); // No change. + }); + + test('field order', function () { + var t = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}] + }); + var field = t.getFields()[0]; + assert.equal(field.getOrder(), 'ascending'); // Default. + }); + + test('compare buffers default order', function () { + var t = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'age', type: 'long'}, + {name: 'name', type: 'string'}, + {name: 'weight', type: 'float'}, + ] + }); + var b1 = t.toBuffer({age: 20, name: 'Ann', weight: 0.5}); + assert.equal(t.compareBuffers(b1, b1), 0); + var b2 = t.toBuffer({age: 20, name: 'Bob', weight: 0}); + assert.equal(t.compareBuffers(b1, b2), -1); + var b3 = t.toBuffer({age: 19, name: 'Carrie', weight: 0}); + assert.equal(t.compareBuffers(b1, b3), 1); + }); + + test('compare buffers custom order', function () { + var t = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'meta', type: {type: 'map', values: 'int'}, order: 'ignore'}, + {name: 'name', type: 'string', order: 'descending'} + ] + }); + var b1 = t.toBuffer({meta: {}, name: 'Ann'}); + assert.equal(t.compareBuffers(b1, b1), 0); + var b2 = t.toBuffer({meta: {foo: 1}, name: 'Bob'}); + assert.equal(t.compareBuffers(b1, b2), 1); + var b3 = t.toBuffer({meta: {foo: 0}, name: 'Alex'}); + assert.equal(t.compareBuffers(b1, b3), -1); + }); + + test('compare buffers invalid order', function () { + assert.throws(function () { createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int', order: 'up'}] + }); }); + }); + + test('error type', function () { + var t = createType({ + type: 'error', + name: 'Ouch', + fields: [{name: 'name', type: 'string'}] + }); + var E = t.getRecordConstructor(); + var err = new E('MyError'); + assert(err instanceof Error); + }); + + test('isValid hook', function () { + var t = createType({ + type: 'record', + name: 'Person', + fields: [ + {name: 'age', type: 'int'}, + {name: 'names', type: {type: 'array', items: 'string'}} + ] + }); + var hasErr = false; + try { + assert(!t.isValid({age: 23, names: ['ann', null]}, {errorHook: hook})); + } catch (err) { + hasErr = true; + } + assert(hasErr); + hasErr = false; + try { + // Again to make sure `PATH` was correctly reset. + assert(!t.isValid({age: 23, names: ['ann', null]}, {errorHook: hook})); + } catch (err) { + hasErr = true; + } + assert(hasErr); + + function hook(path, obj, type) { + assert.strictEqual(type, t.getFields()[1].getType().getItemsType()); + assert.deepEqual(path, ['names', '1']); + throw new Error(); + } + }); + + test('isValid empty record', function () { + var t = createType({type: 'record', name: 'Person', fields: []}); + assert(t.isValid({})); + }); + + }); + + suite('AbstractLongType', function () { + + var fastLongType = new types.LongType(); + + suite('unpacked', function () { + + var slowLongType = types.LongType.using({ + fromBuffer: function (buf) { + var neg = buf[7] >> 7; + if (neg) { // Negative number. + invert(buf); + } + var n = buf.readInt32LE() + Math.pow(2, 32) * buf.readInt32LE(4); + if (neg) { + invert(buf); + n = -n - 1; + } + return n; + }, + toBuffer: function (n) { + var buf = new Buffer(8); + var neg = n < 0; + if (neg) { + invert(buf); + n = -n - 1; + } + buf.writeInt32LE(n | 0); + var h = n / Math.pow(2, 32) | 0; + buf.writeInt32LE(h ? h : (n >= 0 ? 0 : -1), 4); + if (neg) { + invert(buf); + } + return buf; + }, + isValid: function (n) { + return typeof n == 'number' && n % 1 === 0; + }, + fromJSON: function (n) { return n; }, + toJSON: function (n) { return n; }, + compare: function (n1, n2) { + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); + } + }); + + test('encode', function () { + [123, -1, 321414, 900719925474090].forEach(function (n) { + assert.deepEqual(slowLongType.toBuffer(n), fastLongType.toBuffer(n)); + }); + }); + + test('decode', function () { + [123, -1, 321414, 900719925474090].forEach(function (n) { + var buf = fastLongType.toBuffer(n); + assert.deepEqual(slowLongType.fromBuffer(buf), n); + }); + }); + + test('clone', function () { + assert.equal(slowLongType.clone(123), 123); + assert.equal(slowLongType.fromString('-1'), -1); + assert.equal(slowLongType.toString(-1), '-1'); + }); + + test('random', function () { + assert(slowLongType.isValid(slowLongType.random())); + }); + + test('isValid hook', function () { + var s = 'hi'; + var errs = []; + assert(!slowLongType.isValid(s, {errorHook: hook})); + assert.deepEqual(errs, [s]); + assert.throws(function () { slowLongType.toBuffer(s); }); + + function hook(path, obj, type) { + assert.strictEqual(type, slowLongType); + assert.equal(path.length, 0); + errs.push(obj); + } + }); + + }); + + suite('packed', function () { + + var slowLongType = types.LongType.using({ + fromBuffer: function (buf) { + var tap = new Tap(buf); + return tap.readLong(); + }, + toBuffer: function (n) { + var buf = new Buffer(10); + var tap = new Tap(buf); + tap.writeLong(n); + return buf.slice(0, tap.pos); + }, + fromJSON: function (n) { return n; }, + toJSON: function (n) { return n; }, + isValid: function (n) { return typeof n == 'number' && n % 1 === 0; }, + compare: function (n1, n2) { + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); + } + }, true); + + test('encode', function () { + [123, -1, 321414, 900719925474090].forEach(function (n) { + assert.deepEqual(slowLongType.toBuffer(n), fastLongType.toBuffer(n)); + }); + }); + + test('decode', function () { + [123, -1, 321414, 900719925474090].forEach(function (n) { + var buf = fastLongType.toBuffer(n); + assert.deepEqual(slowLongType.fromBuffer(buf), n); + }); + }); + + test('clone', function () { + assert.equal(slowLongType.clone(123), 123); + assert.equal(slowLongType.fromString('-1'), -1); + assert.equal(slowLongType.toString(-1), '-1'); + }); + + test('random', function () { + assert(slowLongType.isValid(slowLongType.random())); + }); + + }); + + test('incomplete buffer', function () { + // Check that `fromBuffer` doesn't get called. + var slowLongType = new types.LongType.using({ + fromBuffer: function () { throw new Error('no'); }, + toBuffer: null, + fromJSON: null, + toJSON: null, + isValid: null, + compare: null + }); + var buf = fastLongType.toBuffer(12314); + assert.deepEqual( + slowLongType.decode(buf.slice(0, 1)), + {value: undefined, offset: -1} + ); + }); + + }); + + suite('LogicalType', function () { + + function DateType(attrs, opts) { + types.LogicalType.call(this, attrs, opts, [types.LongType]); + } + util.inherits(DateType, types.LogicalType); + + DateType.prototype._fromValue = function (val) { return new Date(val); }; + + DateType.prototype._toValue = function (date) { return +date; }; + + DateType.prototype._resolve = function (type) { + if (type instanceof types.StringType) { + return function (str) { return new Date(Date.parse(str)); }; + } else if (type instanceof types.LongType) { + return this.fromValue; + } + }; + + function AgeType(attrs, opts) { + types.LogicalType.call(this, attrs, opts, [types.IntType]); + } + util.inherits(AgeType, types.LogicalType); + + AgeType.prototype._fromValue = function (val) { + if (val < 0) { throw new Error('invalid age'); } + return val; + }; + + AgeType.prototype._toValue = AgeType.prototype._fromValue; + + var logicalTypes = {age: AgeType, date: DateType}; + + test('valid type', function () { + var t = createType({ + type: 'long', + logicalType: 'date' + }, {logicalTypes: logicalTypes}); + assert(t instanceof DateType); + assert(t.getUnderlyingType() instanceof types.LongType); + assert(t.isValid(t.random())); + var d = new Date(123); + assert.equal(t.toString(d), '123'); + assert.deepEqual(t.fromString('123'), d); + assert.deepEqual(t.clone(d), d); + assert.equal(t.compare(d, d), 0); + assert.equal(t.getSchema(), '"long"'); + }); + + test('invalid type', function () { + var attrs = { + type: 'int', + logicalType: 'date' + }; + var t; + t = createType(attrs); // Missing. + assert(t instanceof types.IntType); + t = createType(attrs, {logicalTypes: logicalTypes}); // Invalid. + assert(t instanceof types.IntType); + assert.throws(function () { + createType(attrs, { + logicalTypes: logicalTypes, + assertLogicalType: true + }); + }); + }); + + test('nested types', function () { + var attrs = { + name: 'Person', + type: 'record', + fields: [ + {name: 'age', type: {type: 'int', logicalType: 'age'}}, + {name: 'time', type: {type: 'long', logicalType: 'date'}} + ] + }; + var base = createType(attrs); + var derived = createType(attrs, {logicalTypes: logicalTypes}); + var fields = derived.getFields(); + assert(fields[0].getType() instanceof AgeType); + assert(fields[1].getType() instanceof DateType); + var date = new Date(Date.now()); + var buf = base.toBuffer({age: 12, time: +date}); + var person = derived.fromBuffer(buf); + assert.deepEqual(person.age, 12); + assert.deepEqual(person.time, date); + assert.throws(function () { derived.toBuffer({age: -1, date: date}); }); + }); + + test('recursive', function () { + + function Person(friends) { this.friends = friends || []; } + + function PersonType(attrs, opts) { + types.LogicalType.call(this, attrs, opts); + } + util.inherits(PersonType, types.LogicalType); + + PersonType.prototype._fromValue = function (val) { + return new Person(val.friends); + }; + + PersonType.prototype._toValue = function (val) { return val; }; + + var t = createType({ + type: 'record', + name: 'Person', + namespace: 'earth', + logicalType: 'person', + fields: [ + {name: 'friends', type: {type: 'array', items: 'Person'}}, + ] + }, {logicalTypes: {'person': PersonType}}); + + var p1 = new Person([new Person()]); + var buf = t.toBuffer(p1); + var p2 = t.fromBuffer(buf); + assert(p2 instanceof Person); + assert(p2.friends[0] instanceof Person); + assert.deepEqual(p2, p1); + }); + + test('resolve underlying > logical', function () { + var t1 = createType({type: 'string'}); + var t2 = createType({ + type: 'long', + logicalType: 'date' + }, {logicalTypes: logicalTypes}); + + var d1 = new Date(Date.now()); + var buf = t1.toBuffer('' + d1); + var res = t2.createResolver(t1); + assert.throws(function () { t2.createResolver(createType('float')); }); + var d2 = t2.fromBuffer(buf, res); + assert.deepEqual('' + d2, '' + d1); // Rounding error on date objects. + }); + + test('resolve logical > underlying', function () { + var t1 = createType({ + type: 'long', + logicalType: 'date' + }, {logicalTypes: logicalTypes}); + var t2 = createType({type: 'double'}); // Note long > double too. + + var d = new Date(Date.now()); + var buf = t1.toBuffer(d); + var res = t2.createResolver(t1); + assert.throws(function () { createType('int').createResolver(t1); }); + assert.equal(t2.fromBuffer(buf, res), +d); + }); + + }); + + suite('createType', function () { + + test('null type', function () { + assert.throws(function () { createType(null); }); + }); + + test('unknown types', function () { + assert.throws(function () { createType('a'); }); + assert.throws(function () { createType({type: 'b'}); }); + }); + + test('namespaced type', function () { + var type = createType({ + type: 'record', + name: 'Human', + namespace: 'earth', + fields: [ + { + name: 'id', + type: {type: 'fixed', name: 'Id', size: 2, namespace: 'all'} + }, + { + name: 'alien', + type: { + type: 'record', + name: 'Alien', + namespace: 'all', + fields: [ + {name: 'friend', type: 'earth.Human'}, + {name: 'id', type: 'Id'}, + ] + } + } + ] + }); + assert.equal(type._name, 'earth.Human'); + assert.equal(type._fields[0]._type._name, 'all.Id'); + assert.equal(type._fields[1]._type._name, 'all.Alien'); + }); + + test('wrapped primitive', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'nothing', type: {type: 'null'}}] + }); + assert.strictEqual(type._fields[0]._type.constructor, types.NullType); + }); + + test('fromBuffer truncated', function () { + var type = createType('int'); + assert.throws(function () { + type.fromBuffer(new Buffer([128])); + }); + }); + + test('fromBuffer bad resolver', function () { + var type = createType('int'); + assert.throws(function () { + type.fromBuffer(new Buffer([0]), 123, {}); + }); + }); + + test('fromBuffer trailing', function () { + var type = createType('int'); + assert.throws(function () { + type.fromBuffer(new Buffer([0, 2])); + }); + }); + + test('fromBuffer trailing with resolver', function () { + var type = createType('int'); + var resolver = type.createResolver(createType(['int'])); + assert.equal(type.fromBuffer(new Buffer([0, 2]), resolver), 1); + }); + + test('toBuffer', function () { + var type = createType('int'); + assert.throws(function () { type.toBuffer('abc'); }); + assert.doesNotThrow(function () { type.toBuffer(123); }); + }); + + test('toBuffer and resize', function () { + var type = createType('string'); + assert.deepEqual(type.toBuffer('\x01', 1), new Buffer([2, 1])); + }); + + test('type hook', function () { + var refs = []; + var ts = []; + var o = { + type: 'record', + name: 'Human', + fields: [ + {name: 'age', type: 'int'}, + {name: 'name', type: {type: 'string'}} + ] + }; + createType(o, {typeHook: hook}); + assert.equal(ts.length, 1); + assert.equal(ts[0].getName(), 'Human'); + + function hook(schema, opts) { + if (~refs.indexOf(schema)) { + // Already seen this schema. + return; + } + refs.push(schema); + + var type = createType(schema, opts); + if (type instanceof types.RecordType) { + ts.push(type); + } + return type; + } + }); + + test('type hook invalid return value', function () { + assert.throws(function () { + createType({type: 'int'}, {typeHook: hook}); + }); + + function hook() { return 'int'; } + }); + + test('fingerprint', function () { + var t = createType('int'); + var buf = new Buffer('ef524ea1b91e73173d938ade36c1db32', 'hex'); + assert.deepEqual(t.getFingerprint('md5'), buf); + assert.deepEqual(t.getFingerprint(), buf); + }); + + test('getSchema default', function () { + var type = createType({ + type: 'record', + name: 'Human', + fields: [ + {name: 'id1', type: ['string', 'null'], 'default': ''}, + {name: 'id2', type: ['null', 'string'], 'default': null} + ] + }); + assert.deepEqual( + JSON.parse(type.getSchema()), + { + type: 'record', + name: 'Human', + fields: [ + {name: 'id1', type: ['string', 'null']}, // Stripped defaults. + {name: 'id2', type: ['null', 'string']} + ] + } + ); + }); + + }); + + suite('fromString', function () { + + test('int', function () { + var t = createType('int'); + assert.equal(t.fromString('2'), 2); + assert.throws(function () { t.fromString('"a"'); }); + }); + + test('string', function () { + var t = createType('string'); + assert.equal(t.fromString('"2"'), '2'); + assert.throws(function () { t.fromString('a'); }); + }); + + test('coerce buffers', function () { + var t = createType({ + name: 'Ids', + type: 'record', + fields: [{name: 'id1', type: {name: 'Id1', type: 'fixed', size: 2}}] + }); + var o = {id1: new Buffer([0, 1])}; + var s = '{"id1": "\\u0000\\u0001"}'; + var c = t.fromString(s); + assert.deepEqual(c, o); + assert(c instanceof t.getRecordConstructor()); + }); + + }); + + suite('toString', function () { + + test('int', function () { + var t = createType('int'); + assert.equal(t.toString(2), '2'); + assert.throws(function () { t.toString('a'); }); + }); + + }); + + suite('resolve', function () { + + test('non type', function () { + var t = createType({type: 'map', values: 'int'}); + var obj = {type: 'map', values: 'int'}; + assert.throws(function () { t.createResolver(obj); }); + }); + + test('union to valid union', function () { + var t1 = createType(['int', 'string']); + var t2 = createType(['null', 'string', 'long']); + var resolver = t2.createResolver(t1); + var buf = t1.toBuffer({'int': 12}); + assert.deepEqual(t2.fromBuffer(buf, resolver), {'long': 12}); + }); + + test('union to invalid union', function () { + var t1 = createType(['int', 'string']); + var t2 = createType(['null', 'long']); + assert.throws(function () { t2.createResolver(t1); }); + }); + + test('union to non union', function () { + var t1 = createType(['int', 'long']); + var t2 = createType('long'); + var resolver = t2.createResolver(t1); + var buf = t1.toBuffer({'int': 12}); + assert.equal(t2.fromBuffer(buf, resolver), 12); + buf = new Buffer([4, 0]); + assert.throws(function () { t2.fromBuffer(buf, resolver); }); + }); + + test('union to invalid non union', function () { + var t1 = createType(['int', 'long']); + var t2 = createType('int'); + assert.throws(function() { t2.createResolver(t1); }); + }); + + }); + + suite('type names', function () { + + test('existing', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'so', type: 'Person'}] + }); + assert.strictEqual(type, type._fields[0]._type); + }); + + test('namespaced', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [ + { + name: 'so', + type: { + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}], + namespace: 'a' + } + } + ] + }); + assert.equal(type._name, 'Person'); + assert.equal(type._fields[0]._type._name, 'a.Person'); + }); + + test('redefining', function () { + assert.throws(function () { + createType({ + type: 'record', + name: 'Person', + fields: [ + { + name: 'so', + type: { + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}] + } + } + ] + }); + }); + }); + + test('missing', function () { + assert.throws(function () { + createType({ + type: 'record', + name: 'Person', + fields: [{name: 'so', type: 'Friend'}] + }); + }); + }); + + test('redefining primitive', function () { + assert.throws( // Unqualified. + function () { createType({type: 'fixed', name: 'int', size: 2}); } + ); + assert.throws( // Qualified. + function () { + createType({type: 'fixed', name: 'int', size: 2, namespace: 'a'}); + } + ); + }); + + test('aliases', function () { + var type = createType({ + type: 'record', + name: 'Person', + namespace: 'a', + aliases: ['Human', 'b.Being'], + fields: [{name: 'age', type: 'int'}] + }); + assert.deepEqual(type._aliases, ['a.Human', 'b.Being']); + }); + + test('invalid', function () { + // Name. + assert.throws(function () { + createType({type: 'fixed', name: 'ID$', size: 3}); + }); + // Namespace. + assert.throws(function () { + createType({type: 'fixed', name: 'ID', size: 3, namespace: '1a'}); + }); + // Qualified. + assert.throws(function () { + createType({type: 'fixed', name: 'a.2.ID', size: 3}); + }); + }); + + }); + + suite('decode', function () { + + test('long valid', function () { + var t = createType('long'); + var buf = new Buffer([0, 128, 2, 0]); + var res = t.decode(buf, 1); + assert.deepEqual(res, {value: 128, offset: 3}); + }); + + test('bytes invalid', function () { + var t = createType('bytes'); + var buf = new Buffer([4, 1]); + var res = t.decode(buf, 0); + assert.deepEqual(res, {value: undefined, offset: -1}); + }); + + }); + + suite('encode', function () { + + test('int valid', function () { + var t = createType('int'); + var buf = new Buffer(2); + buf.fill(0); + var n = t.encode(5, buf, 1); + assert.equal(n, 2); + assert.deepEqual(buf, new Buffer([0, 10])); + }); + + test('string invalid', function () { + var t = createType('string'); + var buf = new Buffer(1); + var n = t.encode('\x01\x02', buf, 0); + assert.equal(n, -2); + }); + + test('invalid', function () { + var t = createType('float'); + var buf = new Buffer(2); + assert.throws(function () { t.encode('hi', buf, 0); }); + }); + + }); + + suite('inspect', function () { + + test('type', function () { + assert.equal(createType('int').inspect(), ''); + assert.equal( + createType({type: 'map', values: 'string'}).inspect(), + '' + ); + assert.equal( + createType({type: 'fixed', name: 'Id', size: 2}).inspect(), + '' + ); + }); + + test('field', function () { + var type = createType({ + type: 'record', + name: 'Person', + fields: [{name: 'age', type: 'int'}] + }); + var field = type.getFields()[0]; + assert.equal(field.inspect(), ''); + }); + + test('resolver', function () { + var t1 = createType('int'); + var t2 = createType('double'); + var resolver = t2.createResolver(t1); + assert.equal(resolver.inspect(), ''); + }); + + }); + + test('reset', function () { + types.Type.__reset(0); + var t = createType('string'); + var buf = t.toBuffer('\x01'); + assert.deepEqual(buf, new Buffer([2, 1])); + }); + +}); + +function testType(Type, data, invalidSchemas) { + + data.forEach(function (elem) { + test('roundtrip', function () { + var type = new Type(elem.schema); + elem.valid.forEach(function (v) { + assert(type.isValid(v), '' + v); + var fn = elem.check || assert.deepEqual; + fn(type.fromBuffer(type.toBuffer(v)), v); + fn(type.fromString(type.toString(v), {coerceBuffers: true}), v); + }); + elem.invalid.forEach(function (v) { + assert(!type.isValid(v), '' + v); + assert.throws(function () { type.isValid(v, {errorHook: hook}); }); + assert.throws(function () { type.toBuffer(v); }); + + function hook() { throw new Error(); } + }); + var n = 50; + while (n--) { + // Run a few times to make sure we cover any branches. + assert(type.isValid(type.random())); + } + }); + }); + + test('skip', function () { + data.forEach(function (elem) { + var fn = elem.check || assert.deepEqual; + var items = elem.valid; + if (items.length > 1) { + var type = new Type(elem.schema); + var buf = new Buffer(1024); + var tap = new Tap(buf); + type._write(tap, items[0]); + type._write(tap, items[1]); + tap.pos = 0; + type._skip(tap); + fn(type._read(tap), items[1]); + } + }); + }); + + if (invalidSchemas) { + test('invalid', function () { + invalidSchemas.forEach(function (schema) { + assert.throws(function () { new Type(schema); }); + }); + }); + } + +} + +function getResolver(reader, writer) { + return createType(reader).createResolver(createType(writer)); +} + +function floatEquals(a, b) { + return Math.abs((a - b) / Math.min(a, b)) < 1e-7; +} + +function invert(buf) { + var len = buf.length; + while (len--) { + buf[len] = ~buf[len]; + } +} diff --git a/lang/js/test/test_utils.js b/lang/js/test/test_utils.js new file mode 100644 index 00000000000..9faa1d6319f --- /dev/null +++ b/lang/js/test/test_utils.js @@ -0,0 +1,397 @@ +/* jshint node: true, mocha: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +var utils = require('../lib/utils'), + assert = require('assert'); + + +suite('utils', function () { + + test('capitalize', function () { + assert.equal(utils.capitalize('abc'), 'Abc'); + assert.equal(utils.capitalize(''), ''); + assert.equal(utils.capitalize('aBc'), 'ABc'); + }); + + test('hasDuplicates', function () { + assert(utils.hasDuplicates([1, 3, 1])); + assert(!utils.hasDuplicates([])); + assert(!utils.hasDuplicates(['ab', 'cb'])); + assert(utils.hasDuplicates(['ab', 'cb'], function (s) { return s[1]; })); + }); + + test('single index of', function () { + assert.equal(utils.singleIndexOf(null, 1), -1); + assert.equal(utils.singleIndexOf([2], 2), 0); + assert.equal(utils.singleIndexOf([3, 3], 3), -2); + assert.equal(utils.singleIndexOf([2, 4], 4), 1); + }); + + test('abstract function', function () { + assert.throws(utils.abstractFunction, utils.Error); + }); + + test('OrderedQueue', function () { + + var seqs = [ + [0], + [0,1], + [0,1,2], + [2,1,0], + [0,2,1,3], + [1,3,2,4,0], + [0,1,2,3] + ]; + + var i; + for (i = 0; i < seqs.length; i++) { + check(seqs[i]); + } + + function check(seq) { + var q = new utils.OrderedQueue(); + var i; + assert.strictEqual(q.pop(), null); + for (i = 0; i < seq.length; i++) { + q.push({index: seq[i]}); + } + for (i = 0; i < seq.length; i++) { + var j = q.pop(); + assert.equal(j !== null && j.index, i, seq.join()); + } + } + + }); + + suite('Lcg', function () { + + test('seed', function () { + var r1 = new utils.Lcg(48); + var r2 = new utils.Lcg(48); + assert.equal(r1.nextInt(), r2.nextInt()); + }); + + test('integer', function () { + var r = new utils.Lcg(48); + var i; + i = r.nextInt(); + assert(i >= 0 && i === (i | 0)); + i = r.nextInt(1); + assert.equal(i, 0); + i = r.nextInt(1, 2); + assert.equal(i, 1); + }); + + test('float', function () { + var r = new utils.Lcg(48); + var f; + f = r.nextFloat(); + assert(0 <= f && f < 1); + f = r.nextFloat(0); + assert.equal(f, 0); + f = r.nextFloat(1, 1); + assert.equal(f, 1); + }); + + test('boolean', function () { + var r = new utils.Lcg(48); + assert(typeof r.nextBoolean() == 'boolean'); + }); + + test('choice', function () { + var r = new utils.Lcg(48); + var arr = ['a']; + assert(r.choice(arr), 'a'); + assert.throws(function () { r.choice([]); }); + }); + + test('string', function () { + var r = new utils.Lcg(48); + var s; + s = r.nextString(10, 'aA#!'); + assert.equal(s.length, 10); + s = r.nextString(5, '#!'); + assert.equal(s.length, 5); + }); + + }); + + suite('Tap', function () { + + var Tap = utils.Tap; + + suite('int & long', function () { + + testWriterReader({ + elems: [0, -1, 109213, -1211, -1312411211, 900719925474090], + reader: function () { return this.readLong(); }, + skipper: function () { this.skipLong(); }, + writer: function (n) { this.writeLong(n); } + }); + + test('write', function () { + + var tap = newTap(6); + tap.writeLong(1440756011948); + var buf = new Buffer(['0xd8', '0xce', '0x80', '0xbc', '0xee', '0x53']); + assert(tap.isValid()); + assert(buf.equals(tap.buf)); + + }); + + test('read', function () { + + var buf = new Buffer(['0xd8', '0xce', '0x80', '0xbc', '0xee', '0x53']); + assert.equal((new Tap(buf)).readLong(), 1440756011948); + + }); + + }); + + suite('boolean', function () { + + testWriterReader({ + elems: [true, false], + reader: function () { return this.readBoolean(); }, + skipper: function () { this.skipBoolean(); }, + writer: function (b) { this.writeBoolean(b); } + }); + + }); + + suite('float', function () { + + testWriterReader({ + elems: [1, 3,1, -5, 1e9], + reader: function () { return this.readFloat(); }, + skipper: function () { this.skipFloat(); }, + writer: function (b) { this.writeFloat(b); } + }); + + }); + + suite('double', function () { + + testWriterReader({ + elems: [1, 3,1, -5, 1e12], + reader: function () { return this.readDouble(); }, + skipper: function () { this.skipDouble(); }, + writer: function (b) { this.writeDouble(b); } + }); + + }); + + suite('string', function () { + + testWriterReader({ + elems: ['ahierw', '', 'alh hewlii! rew'], + reader: function () { return this.readString(); }, + skipper: function () { this.skipString(); }, + writer: function (s) { this.writeString(s); } + }); + + }); + + suite('bytes', function () { + + testWriterReader({ + elems: [new Buffer('abc'), new Buffer(0), new Buffer([1, 5, 255])], + reader: function () { return this.readBytes(); }, + skipper: function () { this.skipBytes(); }, + writer: function (b) { this.writeBytes(b); } + }); + + }); + + suite('fixed', function () { + + testWriterReader({ + elems: [new Buffer([1, 5, 255])], + reader: function () { return this.readFixed(3); }, + skipper: function () { this.skipFixed(3); }, + writer: function (b) { this.writeFixed(b, 3); } + }); + + }); + + suite('binary', function () { + + test('write valid', function () { + var tap = newTap(3); + var s = '\x01\x02'; + tap.writeBinary(s, 2); + assert.deepEqual(tap.buf, new Buffer([1,2,0])); + }); + + test('write invalid', function () { + var tap = newTap(1); + var s = '\x01\x02'; + tap.writeBinary(s, 2); + assert.deepEqual(tap.buf, new Buffer([0])); + }); + + }); + + suite('pack & unpack longs', function () { + + test('unpack single byte', function () { + var t = newTap(10); + t.writeLong(5); + t.pos = 0; + assert.deepEqual( + t.unpackLongBytes(), + new Buffer([5, 0, 0, 0, 0, 0, 0, 0]) + ); + t.pos = 0; + t.writeLong(-5); + t.pos = 0; + assert.deepEqual( + t.unpackLongBytes(), + new Buffer([-5, -1, -1, -1, -1, -1, -1, -1]) + ); + t.pos = 0; + }); + + test('unpack multiple bytes', function () { + var t = newTap(10); + var l; + l = 18932; + t.writeLong(l); + t.pos = 0; + assert.deepEqual(t.unpackLongBytes().readInt32LE(), l); + t.pos = 0; + l = -3210984; + t.writeLong(l); + t.pos = 0; + assert.deepEqual(t.unpackLongBytes().readInt32LE(), l); + }); + + test('pack single byte', function () { + var t = newTap(10); + var b = new Buffer(8); + b.fill(0); + b.writeInt32LE(12); + t.packLongBytes(b); + assert.equal(t.pos, 1); + t.pos = 0; + assert.deepEqual(t.readLong(), 12); + t.pos = 0; + b.writeInt32LE(-37); + b.writeInt32LE(-1, 4); + t.packLongBytes(b); + assert.equal(t.pos, 1); + t.pos = 0; + assert.deepEqual(t.readLong(), -37); + t.pos = 0; + b.writeInt32LE(-1); + b.writeInt32LE(-1, 4); + t.packLongBytes(b); + assert.deepEqual(t.buf.slice(0, t.pos), new Buffer([1])); + t.pos = 0; + assert.deepEqual(t.readLong(), -1); + }); + + test('roundtrip', function () { + roundtrip(1231514); + roundtrip(-123); + roundtrip(124124); + roundtrip(109283109271); + roundtrip(Number.MAX_SAFE_INTEGER); + roundtrip(Number.MIN_SAFE_INTEGER); + roundtrip(0); + roundtrip(-1); + + function roundtrip(n) { + var t1 = newTap(10); + var t2 = newTap(10); + t1.writeLong(n); + t1.pos = 0; + t2.packLongBytes(t1.unpackLongBytes()); + assert.deepEqual(t2, t1); + } + }); + + }); + + function newTap(n) { + + var buf = new Buffer(n); + buf.fill(0); + return new Tap(buf); + + } + + function testWriterReader(opts) { + + var size = opts.size; + var elems = opts.elems; + var writeFn = opts.writer; + var readFn = opts.reader; + var skipFn = opts.skipper; + var name = opts.name || ''; + + test('write read ' + name, function () { + var tap = newTap(size || 1024); + var i, l, elem; + for (i = 0, l = elems.length; i < l; i++) { + tap.buf.fill(0); + tap.pos = 0; + elem = elems[i]; + writeFn.call(tap, elem); + tap.pos = 0; + assert.deepEqual(readFn.call(tap), elem); + } + }); + + test('read over ' + name, function () { + var tap = new Tap(new Buffer(0)); + readFn.call(tap); // Shouldn't throw. + assert(!tap.isValid()); + }); + + test('write over ' + name, function () { + var tap = new Tap(new Buffer(0)); + writeFn.call(tap, elems[0]); // Shouldn't throw. + assert(!tap.isValid()); + }); + + test('skip ' + name, function () { + var tap = newTap(size || 1024); + var i, l, elem, pos; + for (i = 0, l = elems.length; i < l; i++) { + tap.buf.fill(0); + tap.pos = 0; + elem = elems[i]; + writeFn.call(tap, elem); + pos = tap.pos; + tap.pos = 0; + skipFn.call(tap, elem); + assert.equal(tap.pos, pos); + } + }); + + } + + }); + +});